From 7b3287f67b25e3e1ba05e0082fb7dbe5a708b6e7 Mon Sep 17 00:00:00 2001 From: Kelly McDonald Date: Tue, 4 May 2021 11:32:34 -0400 Subject: [PATCH 01/13] Save a sample file while I try to figure out how the JS is going to work --- crc/static/jinja_extensions.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 crc/static/jinja_extensions.py diff --git a/crc/static/jinja_extensions.py b/crc/static/jinja_extensions.py new file mode 100644 index 00000000..fcbc7b55 --- /dev/null +++ b/crc/static/jinja_extensions.py @@ -0,0 +1,6 @@ +from crc.api.file import get_document_directory + + +def render_files(study_id,irb_codes): + files = get_document_directory(study_id) + print(files) \ No newline at end of file From cbd1d01203e789d4c6b5f49f73fddd41275cdaa2 Mon Sep 17 00:00:00 2001 From: Kelly McDonald Date: Wed, 5 May 2021 11:30:08 -0400 Subject: [PATCH 02/13] Add URL to the study_info('documents') script fixes #321 - I merged in branches that fix #320 and #297 320-add-default-for-file-data-get 297-filename-in-documents --- crc/api.yml | 35 +++++++++++++++++++ crc/api/file.py | 17 +++++++++ crc/services/study_service.py | 12 +++++++ .../data/file_data_store/file_data_store.bpmn | 4 +++ tests/test_file_datastore.py | 2 ++ 5 files changed, 70 insertions(+) diff --git a/crc/api.yml b/crc/api.yml index 980d6d13..81d45f3a 100644 --- a/crc/api.yml +++ b/crc/api.yml @@ -705,6 +705,41 @@ paths: type: string format: binary example: '' + /file/{file_id}/download : + parameters : + - name : file_id + in : path + required : true + description : The id of the File requested + schema : + type : integer + - name : auth_token + in : query + required : true + description : User Auth Toeken + schema : + type : string + - name : version + in : query + required : false + description : The version of the file, or none for latest version + schema : + type : integer + get : + operationId : crc.api.file.get_file_data_link + summary : Returns only the file contents + security: [] + tags : + - Files + responses : + '200' : + description : Returns the actual file + content : + application/octet-stream : + schema : + type : string + format : binary + example : '' /file/{file_id}/data: parameters: - name: file_id diff --git a/crc/api/file.py b/crc/api/file.py index 743d6327..619fd7ec 100644 --- a/crc/api/file.py +++ b/crc/api/file.py @@ -6,6 +6,7 @@ from flask import send_file from crc import session from crc.api.common import ApiError +from crc.api.user import verify_token from crc.models.api_models import DocumentDirectory, DocumentDirectorySchema from crc.models.file import FileSchema, FileModel, File, FileModelSchema, FileDataModel, FileType from crc.models.workflow import WorkflowSpecModel @@ -179,6 +180,22 @@ def get_file_data(file_id, version=None): ) +def get_file_data_link(file_id, auth_token, version=None): + if not verify_token(auth_token): + raise ApiError('not_authenticated', 'You need to include an authorization token in the URL with this') + file_data = FileService.get_file_data(file_id, version) + if file_data is None: + raise ApiError('no_such_file', 'The file id you provided does not exist') + return send_file( + io.BytesIO(file_data.data), + attachment_filename=file_data.file_model.name, + mimetype=file_data.file_model.content_type, + cache_timeout=-1, # Don't cache these files on the browser. + last_modified=file_data.date_created, + as_attachment = True + ) + + def get_file_info(file_id): file_model = session.query(FileModel).filter_by(id=file_id).with_for_update().first() if file_model is None: diff --git a/crc/services/study_service.py b/crc/services/study_service.py index 7774d2cc..4aa7a0a5 100644 --- a/crc/services/study_service.py +++ b/crc/services/study_service.py @@ -1,7 +1,9 @@ +import urllib from copy import copy from datetime import datetime from typing import List +import flask import requests from SpiffWorkflow import WorkflowException from SpiffWorkflow.exceptions import WorkflowTaskExecException @@ -288,9 +290,19 @@ class StudyService(object): doc_files = FileService.get_files_for_study(study_id=study_id, irb_doc_code=code) doc['count'] = len(doc_files) doc['files'] = [] + + # when we run tests - it doesn't look like the user is available + # so we return a bogus token + token = 'not_available' + if hasattr(flask.g,'user'): + token = flask.g.user.encode_auth_token() for file in doc_files: doc['files'].append({'file_id': file.id, 'name': file.name, + 'url': app.config['APPLICATION_ROOT']+ + 'file/' + str(file.id) + + '/download?auth_token='+ + urllib.parse.quote_plus(token), 'workflow_id': file.workflow_id}) # update the document status to match the status of the workflow it is in. diff --git a/tests/data/file_data_store/file_data_store.bpmn b/tests/data/file_data_store/file_data_store.bpmn index 9af77bb0..ddc9e987 100644 --- a/tests/data/file_data_store/file_data_store.bpmn +++ b/tests/data/file_data_store/file_data_store.bpmn @@ -18,6 +18,10 @@ fileid = documents['UVACompl_PRCAppr'].files[0]['file_id'] +fileurl = documents['UVACompl_PRCAppr'].files[0]['url'] + +filename = documents['UVACompl_PRCAppr'].files[0]['name'] + file_data_set(file_id=fileid,key='test',value='me') diff --git a/tests/test_file_datastore.py b/tests/test_file_datastore.py index 2b95519e..1a10eea7 100644 --- a/tests/test_file_datastore.py +++ b/tests/test_file_datastore.py @@ -27,6 +27,8 @@ class TestFileDatastore(BaseTest): processor = WorkflowProcessor(workflow) processor.do_engine_steps() task_data = processor.bpmn_workflow.last_task.data + self.assertTrue(str(task_data['fileid']) in task_data['fileurl']) + self.assertEqual(task_data['filename'],'anything.png') self.assertEqual(task_data['output'], 'me') self.assertEqual(task_data['output2'], 'nope') From 6b242c07e29a236a62500788b597812e9741c9f4 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 11:54:04 -0400 Subject: [PATCH 03/13] Set the default for the `standalone` parameter to `false`. --- crc/api.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/crc/api.yml b/crc/api.yml index 2e36b93a..49728a77 100644 --- a/crc/api.yml +++ b/crc/api.yml @@ -1574,6 +1574,7 @@ components: standalone: type: boolean example: false + default: false workflow_spec_category: $ref: "#/components/schemas/WorkflowSpecCategory" is_status: From 09a395fa34e61905135fbd64b66cd7d6f3acba39 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 14:07:21 -0400 Subject: [PATCH 04/13] Start for ticket 230 - Get Dashboard URL --- config/default.py | 1 + crc/scripts/get_dashboard_url.py | 16 ++++++++++++++++ tests/test_get_dashboard_url_script.py | 7 +++++++ 3 files changed, 24 insertions(+) create mode 100644 crc/scripts/get_dashboard_url.py create mode 100644 tests/test_get_dashboard_url_script.py diff --git a/config/default.py b/config/default.py index de512b38..d9af6a76 100644 --- a/config/default.py +++ b/config/default.py @@ -17,6 +17,7 @@ API_TOKEN = environ.get('API_TOKEN', default = 'af95596f327c9ecc007b60414fc84b61 NAME = "CR Connect Workflow" DEFAULT_PORT = "5000" FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default=DEFAULT_PORT) +FRONTEND = "localhost:4200" CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default="localhost:4200, localhost:5002")) TESTING = environ.get('TESTING', default="false") == "true" PRODUCTION = (environ.get('PRODUCTION', default="false") == "true") diff --git a/crc/scripts/get_dashboard_url.py b/crc/scripts/get_dashboard_url.py new file mode 100644 index 00000000..d43971dd --- /dev/null +++ b/crc/scripts/get_dashboard_url.py @@ -0,0 +1,16 @@ +from crc.scripts.script import Script +from crc import app + + +class GetDashboardURL(Script): + + def get_description(self): + """Get the URL for the main dashboard. This should be system instance aware. + I.e., dev, testing, production, etc.""" + + def do_task_validate_only(self, task, study_id, workflow_id, *args, **kwargs): + self.do_task(task, study_id, workflow_id, *args, **kwargs) + + def do_task(self, task, study_id, workflow_id, *args, **kwargs): + frontend = app.config['FRONTEND'] + return f'http://{frontend}' diff --git a/tests/test_get_dashboard_url_script.py b/tests/test_get_dashboard_url_script.py new file mode 100644 index 00000000..a464400d --- /dev/null +++ b/tests/test_get_dashboard_url_script.py @@ -0,0 +1,7 @@ +from tests.base_test import BaseTest + + +class TestGetDashboardURL(BaseTest): + + def test_get_dashboard_url(self): + pass From 8d698cf27516cd3024e5a88b1521e08a83dfcc35 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 15:31:33 -0400 Subject: [PATCH 05/13] Added explicit config values for `FRONTEND` and `BPMN` hostname and port. Use these new values to define `CORS_ALLOW_ORIGINS` Removed unused `FRONTEND_AUTH_CALLBACK` --- config/default.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/default.py b/config/default.py index d9af6a76..464b4ef7 100644 --- a/config/default.py +++ b/config/default.py @@ -18,7 +18,9 @@ NAME = "CR Connect Workflow" DEFAULT_PORT = "5000" FLASK_PORT = environ.get('PORT0') or environ.get('FLASK_PORT', default=DEFAULT_PORT) FRONTEND = "localhost:4200" -CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default="localhost:4200, localhost:5002")) +BPMN = "localhost:5002" +CORS_DEFAULT = f'{FRONTEND}, {BPMN}' +CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default=CORS_DEFAULT)) TESTING = environ.get('TESTING', default="false") == "true" PRODUCTION = (environ.get('PRODUCTION', default="false") == "true") TEST_UID = environ.get('TEST_UID', default="dhf8r") @@ -51,7 +53,6 @@ SQLALCHEMY_DATABASE_URI = environ.get( TOKEN_AUTH_TTL_HOURS = float(environ.get('TOKEN_AUTH_TTL_HOURS', default=24)) SECRET_KEY = environ.get('SECRET_KEY', default="Shhhh!!! This is secret! And better darn well not show up in prod.") -FRONTEND_AUTH_CALLBACK = environ.get('FRONTEND_AUTH_CALLBACK', default="http://localhost:4200/session") SWAGGER_AUTH_KEY = environ.get('SWAGGER_AUTH_KEY', default="SWAGGER") # %s/%i placeholders expected for uva_id and study_id in various calls. PB_ENABLED = environ.get('PB_ENABLED', default="false") == "true" From 375089e5dfcff6f1618ba8c8ddbfcecb0afe600a Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 15:32:28 -0400 Subject: [PATCH 06/13] test and bpmn file for new get_dashboard_url script --- .../email_dashboard_url.bpmn | 53 +++++++++++++++++++ tests/test_get_dashboard_url_script.py | 12 ++++- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tests/data/email_dashboard_url/email_dashboard_url.bpmn diff --git a/tests/data/email_dashboard_url/email_dashboard_url.bpmn b/tests/data/email_dashboard_url/email_dashboard_url.bpmn new file mode 100644 index 00000000..ce75e589 --- /dev/null +++ b/tests/data/email_dashboard_url/email_dashboard_url.bpmn @@ -0,0 +1,53 @@ + + + + + Flow_0c51a4b + + + + + Flow_1rfvzi5 + + + + Flow_0c51a4b + Flow_1ker6ik + dashboard_url = get_dashboard_url() + + + <a href="{{dashboard_url}}">{{dashboard_url}}</a> + Flow_1ker6ik + Flow_1rfvzi5 + email(subject='My Email Subject', recipients="test@example.com") + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_get_dashboard_url_script.py b/tests/test_get_dashboard_url_script.py index a464400d..988c4909 100644 --- a/tests/test_get_dashboard_url_script.py +++ b/tests/test_get_dashboard_url_script.py @@ -1,7 +1,17 @@ from tests.base_test import BaseTest +from crc import app, mail class TestGetDashboardURL(BaseTest): def test_get_dashboard_url(self): - pass + with mail.record_messages() as outbox: + + dashboard_url = f'http://{app.config["FRONTEND"]}' + workflow = self.create_workflow('email_dashboard_url') + self.get_workflow_api(workflow) + + self.assertEqual(1, len(outbox)) + self.assertEqual('My Email Subject', outbox[0].subject) + self.assertEqual(['test@example.com'], outbox[0].recipients) + self.assertIn(dashboard_url, outbox[0].body) From 5efb1a72ce7b5e24fd288dcd2986f3330d6d1d57 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 15:52:27 -0400 Subject: [PATCH 07/13] Changed URL to use HTTPS instead of HTTP --- crc/scripts/get_dashboard_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crc/scripts/get_dashboard_url.py b/crc/scripts/get_dashboard_url.py index d43971dd..7730dab0 100644 --- a/crc/scripts/get_dashboard_url.py +++ b/crc/scripts/get_dashboard_url.py @@ -13,4 +13,4 @@ class GetDashboardURL(Script): def do_task(self, task, study_id, workflow_id, *args, **kwargs): frontend = app.config['FRONTEND'] - return f'http://{frontend}' + return f'https://{frontend}' From 4dc4e270aa0163cc4fb893f4ac6c6c001ba896a2 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Thu, 6 May 2021 15:54:39 -0400 Subject: [PATCH 08/13] Fixed the test to also use HTTPS --- tests/test_get_dashboard_url_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_get_dashboard_url_script.py b/tests/test_get_dashboard_url_script.py index 988c4909..217942f6 100644 --- a/tests/test_get_dashboard_url_script.py +++ b/tests/test_get_dashboard_url_script.py @@ -7,7 +7,7 @@ class TestGetDashboardURL(BaseTest): def test_get_dashboard_url(self): with mail.record_messages() as outbox: - dashboard_url = f'http://{app.config["FRONTEND"]}' + dashboard_url = f'https://{app.config["FRONTEND"]}' workflow = self.create_workflow('email_dashboard_url') self.get_workflow_api(workflow) From 07f3d2c2ee64526bef04eed96be7adbf794f22ad Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 7 May 2021 14:06:53 -0400 Subject: [PATCH 09/13] Allow data from read-only fields to persist in task_data --- crc/services/workflow_service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crc/services/workflow_service.py b/crc/services/workflow_service.py index 101929d4..a76553ce 100644 --- a/crc/services/workflow_service.py +++ b/crc/services/workflow_service.py @@ -739,10 +739,7 @@ class WorkflowService(object): if hasattr(task.task_spec, 'form'): for field in task.task_spec.form.fields: - if field.has_property(Task.FIELD_PROP_READ_ONLY) and \ - field.get_property(Task.FIELD_PROP_READ_ONLY).lower().strip() == "true": - continue # Don't add read-only data - elif field.has_property(Task.FIELD_PROP_REPEAT): + if field.has_property(Task.FIELD_PROP_REPEAT): group = field.get_property(Task.FIELD_PROP_REPEAT) if group in latest_data: data[group] = latest_data[group] From 67321ecd977869fafbe9894f31f697e057cd0f44 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Fri, 7 May 2021 14:07:19 -0400 Subject: [PATCH 10/13] Test and workflow for persisting read-only data --- .../data/read_only_field/read_only_field.bpmn | 77 +++++++++++++++++++ .../workflow/test_workflow_read_only_field.py | 16 ++++ 2 files changed, 93 insertions(+) create mode 100644 tests/data/read_only_field/read_only_field.bpmn create mode 100644 tests/workflow/test_workflow_read_only_field.py diff --git a/tests/data/read_only_field/read_only_field.bpmn b/tests/data/read_only_field/read_only_field.bpmn new file mode 100644 index 00000000..3620afdf --- /dev/null +++ b/tests/data/read_only_field/read_only_field.bpmn @@ -0,0 +1,77 @@ + + + + + Flow_0to8etb + + + + + + Flow_0a95kns + + + + Flow_0to8etb + Flow_04r75ca + string_value = 'asdf' + + + + + + + + + + + + + Flow_04r75ca + Flow_0g25v76 + + + Read only is {{ read_only_field }} + Flow_0g25v76 + Flow_0a95kns + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/workflow/test_workflow_read_only_field.py b/tests/workflow/test_workflow_read_only_field.py new file mode 100644 index 00000000..804650d8 --- /dev/null +++ b/tests/workflow/test_workflow_read_only_field.py @@ -0,0 +1,16 @@ +from tests.base_test import BaseTest + + +class TestReadOnlyField(BaseTest): + + def test_read_only(self): + + workflow = self.create_workflow('read_only_field') + workflow_api = self.get_workflow_api(workflow) + first_task = workflow_api.next_task + read_only_field = first_task.data['read_only_field'] + self.complete_form(workflow, first_task, {'read_only_field': read_only_field}) + workflow_api = self.get_workflow_api(workflow) + task = workflow_api.next_task + + self.assertEqual('Read only is asdf', task.documentation) \ No newline at end of file From 8a6bef5af40cfa8893f398a492d6750f9a30545d Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 12 May 2021 13:51:51 -0400 Subject: [PATCH 11/13] New service to get the `primary workflow` for a workflow_spec, using a workflow_spec_id. This is necessary because `primary` is a file parameter, not a workflow_spec parameter. - you can ask a workflow file whether it is primary, but - you cannot ask a workflow_spec for its primary workflow file Now, you can use `workflow_service.get_primary_workflow(workflow_spec_id)` --- crc/services/workflow_service.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crc/services/workflow_service.py b/crc/services/workflow_service.py index 101929d4..d0c314f1 100644 --- a/crc/services/workflow_service.py +++ b/crc/services/workflow_service.py @@ -22,7 +22,7 @@ from jinja2 import Template from crc import db, app from crc.api.common import ApiError from crc.models.api_models import Task, MultiInstanceType, WorkflowApi -from crc.models.file import LookupDataModel +from crc.models.file import LookupDataModel, FileModel from crc.models.study import StudyModel from crc.models.task_event import TaskEventModel from crc.models.user import UserModel, UserModelSchema @@ -811,3 +811,12 @@ class WorkflowService(object): def get_standalone_workflow_specs(): specs = db.session.query(WorkflowSpecModel).filter_by(standalone=True).all() return specs + + @staticmethod + def get_primary_workflow(workflow_spec_id): + # Returns the FileModel of the primary workflow for a workflow_spec + primary = None + file = db.session.query(FileModel).filter(FileModel.workflow_spec_id==workflow_spec_id, FileModel.primary==True).first() + if file: + primary = file + return primary From 8f8d9d30e43123610b81092bae28f126ae5d2f7b Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 12 May 2021 13:52:18 -0400 Subject: [PATCH 12/13] Added test for get_primary_workflow --- tests/workflow/test_workflow_service.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/workflow/test_workflow_service.py b/tests/workflow/test_workflow_service.py index a4af4edc..dc6eb01e 100644 --- a/tests/workflow/test_workflow_service.py +++ b/tests/workflow/test_workflow_service.py @@ -10,6 +10,7 @@ from example_data import ExampleDataLoader from crc import db from crc.models.task_event import TaskEventModel from crc.models.api_models import Task +from crc.models.file import FileModel from crc.api.common import ApiError @@ -114,3 +115,12 @@ class TestWorkflowService(BaseTest): result2 = WorkflowService.get_dot_value(path, {"a.b.c":"garbage"}) self.assertEqual("garbage", result2) + + def test_get_primary_workflow(self): + + workflow = self.create_workflow('hello_world') + workflow_spec_id = workflow.workflow_spec.id + primary_workflow = WorkflowService.get_primary_from_workflow_spec(workflow_spec_id) + self.assertIsInstance(primary_workflow, FileModel) + self.assertEqual(workflow_spec_id, primary_workflow.workflow_spec_id) + self.assertEqual('hello_world.bpmn', primary_workflow.name) From 4c8ea144357133bba01ffc921057b21cdb93baa1 Mon Sep 17 00:00:00 2001 From: mike cullerton Date: Wed, 12 May 2021 14:07:54 -0400 Subject: [PATCH 13/13] Fixed test - changed the method name --- tests/workflow/test_workflow_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/workflow/test_workflow_service.py b/tests/workflow/test_workflow_service.py index dc6eb01e..cb4da9c2 100644 --- a/tests/workflow/test_workflow_service.py +++ b/tests/workflow/test_workflow_service.py @@ -120,7 +120,7 @@ class TestWorkflowService(BaseTest): workflow = self.create_workflow('hello_world') workflow_spec_id = workflow.workflow_spec.id - primary_workflow = WorkflowService.get_primary_from_workflow_spec(workflow_spec_id) + primary_workflow = WorkflowService.get_primary_workflow(workflow_spec_id) self.assertIsInstance(primary_workflow, FileModel) self.assertEqual(workflow_spec_id, primary_workflow.workflow_spec_id) self.assertEqual('hello_world.bpmn', primary_workflow.name)