From ae86ab4790010622f77baaa88db214abd1517573 Mon Sep 17 00:00:00 2001
From: mike cullerton <michaelc@cullerton.com>
Date: Tue, 25 Jan 2022 12:18:29 -0500
Subject: [PATCH] Make erroring workflows visible The
 `process_erroring_workflows` method is called by our scheduler in __init__.py

---
 crc/__init__.py                  |  1 +
 crc/services/workflow_service.py | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/crc/__init__.py b/crc/__init__.py
index ebd7e0a5..1dbae32d 100644
--- a/crc/__init__.py
+++ b/crc/__init__.py
@@ -67,6 +67,7 @@ def process_waiting_tasks():
 def init_scheduler():
     scheduler.add_job(process_waiting_tasks, 'interval', minutes=1)
     scheduler.add_job(FileService.cleanup_file_data, 'interval', minutes=1440)  # once a day
+    scheduler.add_job(WorkflowService().process_erroring_workflows, 'interval', minutes=1440)
     scheduler.start()
 
 
diff --git a/crc/services/workflow_service.py b/crc/services/workflow_service.py
index d500c2ab..0ab77f0c 100755
--- a/crc/services/workflow_service.py
+++ b/crc/services/workflow_service.py
@@ -39,6 +39,9 @@ from crc.services.study_service import StudyService
 from crc.services.user_service import UserService
 from crc.services.workflow_processor import WorkflowProcessor
 
+from flask import request
+from sentry_sdk import capture_message, push_scope
+
 
 class WorkflowService(object):
     TASK_ACTION_COMPLETE = "COMPLETE"
@@ -118,6 +121,43 @@ class WorkflowService(object):
                                   workflow_model.study_id,
                                   str(e)))
 
+    @staticmethod
+    def get_erroring_workflows():
+        workflows = session.query(WorkflowModel).filter(WorkflowModel.status==WorkflowStatus.erroring).all()
+        return workflows
+
+    @staticmethod
+    def get_workflow_url(workflow):
+        base_url = app.config['FRONTEND']
+        workflow_url = f'https://{base_url}/workflow/{workflow.id}'
+        return workflow_url
+
+    def process_erroring_workflows(self):
+        workflows = self.get_erroring_workflows()
+        if len(workflows) > 0:
+            workflow_urls = []
+            if len(workflows) == 1:
+                workflow = workflows[0]
+                workflow_url_link = self.get_workflow_url(workflow)
+                workflow_urls.append(workflow_url_link)
+                message = 'There is one workflow in an error state.'
+                message += f'\n You can restart the workflow at {workflow_url_link}.'
+            else:
+                message = f'There are {len(workflows)} workflows in an error state.'
+                message += '\nYou can restart the workflows at these URLs:'
+                for workflow in workflows:
+                    workflow_url_link = self.get_workflow_url(workflow)
+                    workflow_urls.append(workflow_url_link)
+                    message += f'\n{workflow_url_link}'
+
+            with push_scope() as scope:
+                scope.user = {"urls": workflow_urls}
+                scope.set_extra("workflow_urls", workflow_urls)
+                # this sends a message through sentry
+                capture_message(message)
+            # We return message so we can use it in a test
+            return message
+
     @staticmethod
     def raise_if_disabled(spec_id, study_id):
         """Raise an exception of the workflow is not enabled and can not be executed."""