Make erroring workflows visible

The `process_erroring_workflows` method is called by our scheduler in __init__.py

crc/__init__.py

@@ -67,6 +67,7 @@ def process_waiting_tasks():
 def init_scheduler():
     scheduler.add_job(process_waiting_tasks, 'interval', minutes=1)
     scheduler.add_job(FileService.cleanup_file_data, 'interval', minutes=1440)  # once a day
+    scheduler.add_job(WorkflowService().process_erroring_workflows, 'interval', minutes=1440)
     scheduler.start()
 
 

crc/services/workflow_service.py

@@ -39,6 +39,9 @@ from crc.services.study_service import StudyService
 from crc.services.user_service import UserService
 from crc.services.workflow_processor import WorkflowProcessor
 
+from flask import request
+from sentry_sdk import capture_message, push_scope
+
 
 class WorkflowService(object):
     TASK_ACTION_COMPLETE = "COMPLETE"
@@ -118,6 +121,43 @@ class WorkflowService(object):
                                   workflow_model.study_id,
                                   str(e)))
 
+    @staticmethod
+    def get_erroring_workflows():
+        workflows = session.query(WorkflowModel).filter(WorkflowModel.status==WorkflowStatus.erroring).all()
+        return workflows
+
+    @staticmethod
+    def get_workflow_url(workflow):
+        base_url = app.config['FRONTEND']
+        workflow_url = f'https://{base_url}/workflow/{workflow.id}'
+        return workflow_url
+
+    def process_erroring_workflows(self):
+        workflows = self.get_erroring_workflows()
+        if len(workflows) > 0:
+            workflow_urls = []
+            if len(workflows) == 1:
+                workflow = workflows[0]
+                workflow_url_link = self.get_workflow_url(workflow)
+                workflow_urls.append(workflow_url_link)
+                message = 'There is one workflow in an error state.'
+                message += f'\n You can restart the workflow at {workflow_url_link}.'
+            else:
+                message = f'There are {len(workflows)} workflows in an error state.'
+                message += '\nYou can restart the workflows at these URLs:'
+                for workflow in workflows:
+                    workflow_url_link = self.get_workflow_url(workflow)
+                    workflow_urls.append(workflow_url_link)
+                    message += f'\n{workflow_url_link}'
+
+            with push_scope() as scope:
+                scope.user = {"urls": workflow_urls}
+                scope.set_extra("workflow_urls", workflow_urls)
+                # this sends a message through sentry
+                capture_message(message)
+            # We return message so we can use it in a test
+            return message
+
     @staticmethod
     def raise_if_disabled(spec_id, study_id):
         """Raise an exception of the workflow is not enabled and can not be executed."""