performance improvements. and last rements of load_example_data()

crc/api/workflow.py

@@ -1,3 +1,4 @@
+import time
 import uuid
 
 from flask import g
@@ -104,6 +105,7 @@ def drop_workflow_spec_library(spec_id, library_id):
 
 
 def validate_workflow_specification(spec_id, study_id=None, test_until=None):
+
     try:
         master_spec = WorkflowSpecService().master_spec
         if study_id is not None:
@@ -112,8 +114,16 @@ def validate_workflow_specification(spec_id, study_id=None, test_until=None):
             if spec_id in statuses and statuses[spec_id]['status'] == 'disabled':
                 raise ApiError(code='disabled_workflow',
                                message=f"This workflow is disabled. {statuses[spec_id]['message']}")
+        ts = time.time()
         WorkflowService.test_spec(spec_id, study_id, test_until)
+        te = time.time()
+        print('| %2.4f | % s ' % (te - ts, 'validate and complete all fields'))
+
+        ts = time.time()
         WorkflowService.test_spec(spec_id, study_id, test_until, required_only=True)
+        te = time.time()
+        print('| %2.4f | % s ' % (te - ts, 'validate only with required fields'))
+
     except ApiError as ae:
         error = ae
         error = ValidationErrorService.interpret_validation_error(error)

crc/services/document_service.py

@@ -1,3 +1,5 @@
+import time
+
 from crc import session
 from crc.api.common import ApiError
 from crc.models.api_models import DocumentDirectory

crc/services/lookup_service.py

@@ -54,8 +54,8 @@ class LookupService(object):
     def get_lookup_model_for_reference(file_name, value_column, label_column):
         lookup_model = db.session.query(LookupFileModel).\
             filter(LookupFileModel.file_name == file_name). \
-            filter(LookupFileModel.workflow_spec_id is None).\
-            first()
+            filter(LookupFileModel.workflow_spec_id == None).\
+            first()   # use "==" not "is none" which does NOT work, and makes this constantly expensive.
         if not lookup_model:
             logging.warning("!!!! Making a very expensive call to update the lookup model.")
             file_data = ReferenceFileService().get_data(file_name)

crc/services/workflow_service.py

@@ -1,6 +1,7 @@
 import copy
 import json
 import sys
+import time
 import traceback
 import random
 import string
@@ -144,6 +145,7 @@ class WorkflowService(object):
 
             while not processor.bpmn_workflow.is_completed():
                 processor.bpmn_workflow.get_deep_nav_list()  # Assure no errors with navigation.
+
                 exit_task = processor.bpmn_workflow.do_engine_steps(exit_at=test_until)
                 if (exit_task != None):
                     raise ApiError.from_task("validation_break",
@@ -201,6 +203,7 @@ class WorkflowService(object):
         form_data = data
 
         hide_groups = []
+
         for field in task_api.form.fields:
             # Assure we have a field type
             if field.type is None:
@@ -292,6 +295,7 @@ class WorkflowService(object):
                         form_data[group][i][field.id] = WorkflowService.get_random_data_for_field(field, task)
             else:
                 form_data[field.id] = WorkflowService.get_random_data_for_field(field, task)
+
         if task.data is None:
             task.data = {}
 

tests/base_test.py

@@ -146,13 +146,6 @@ class BaseTest(unittest.TestCase):
         """
         ExampleDataLoader.clean_db()
 
-    def load_example_data(self, use_crc_data=False, use_rrt_data=False):
-        """use_crc_data will cause this to load the mammoth collection of documents
-        we built up developing crc, use_rrt_data will do the same for hte rrt project,
-         otherwise it depends on a small setup for running tests."""
-#        from example_data import ExampleDataLoader
-        pass
-
     def add_users(self):
         for user_json in self.users:
             ldap_info = LdapService.user_info(user_json['uid'])

tests/files/test_spec_files_api.py

@@ -18,7 +18,6 @@ from sqlalchemy import column
 class TestFilesApi(BaseTest):
 
     def test_list_files_for_workflow_spec(self):
-        self.load_example_data(use_crc_data=True)
         spec_id = 'random_fact'
         spec = self.load_test_spec(spec_id)
         rv = self.app.get('/v1.0/workflow-specification/%s/file' % spec_id,

tests/study/test_study_data_store_script.py

@@ -22,9 +22,6 @@ class TestSudySponsorsScript(BaseTest):
         mock_get.return_value.text = self.protocol_builder_response('sponsors.json')
         app.config['PB_ENABLED'] = True
         flask.g.user = UserModel(uid='dhf8r')
-        self.load_example_data() # study_info script complains if irb_documents.xls is not loaded
-                                 # during the validate phase I'm going to assume that we will never
-                                 # have a case where irb_documents.xls is not loaded ??
 
         self.load_test_spec("study_sponsors_data_store")
         WorkflowService.test_spec("study_sponsors_data_store")  # This would raise errors if it didn't validate

tests/workflow/test_workflow_spec_validation_api.py

@@ -16,7 +16,6 @@ from crc.services.workflow_service import WorkflowService
 class TestWorkflowSpecValidation(BaseTest):
 
     def validate_workflow(self, workflow_name):
-        # ts = time.time()
         spec_model = self.load_test_spec(workflow_name)
         total_workflows = session.query(WorkflowModel).count()
         rv = self.app.get('/v1.0/workflow-specification/%s/validate' % spec_model.id, headers=self.logged_in_headers())
@@ -24,11 +23,15 @@ class TestWorkflowSpecValidation(BaseTest):
         total_workflows_after = session.query(WorkflowModel).count()
         self.assertEqual(total_workflows, total_workflows_after, "No rogue workflow exists after validation.")
         json_data = json.loads(rv.get_data(as_text=True))
-        # te = time.time()
-        # print('| %2.4f | % s ' % (te - ts, workflow_name))
 
         return ApiErrorSchema(many=True).load(json_data)
 
+    def test_stupid_slow_file_upload_form(self):
+        self.load_test_spec('empty_workflow', master_spec=True)
+        self.create_reference_document()
+        self.assertEqual(0, len(self.validate_workflow("file_upload_form")))
+
+
     def test_successful_validation_of_test_workflows(self):
         app.config['PB_ENABLED'] = False  # Assure this is disabled.
         self.load_test_spec('empty_workflow', master_spec=True)
@@ -37,7 +40,6 @@ class TestWorkflowSpecValidation(BaseTest):
         self.assertEqual(0, len(self.validate_workflow("decision_table")))
         self.assertEqual(0, len(self.validate_workflow("docx")))
         self.assertEqual(0, len(self.validate_workflow("exclusive_gateway")))
-        self.assertEqual(0, len(self.validate_workflow("file_upload_form")))
         self.assertEqual(0, len(self.validate_workflow("random_fact")))
         self.assertEqual(0, len(self.validate_workflow("study_details")))
         self.assertEqual(0, len(self.validate_workflow("two_forms")))