status-im/cabot
2
0
Fork 0
mirror of https://github.com/status-im/cabot.git synced 2025-02-24 10:28:06 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #442 from arachnys/trigger-alert-severity

Browse Source 
Make alerts trigger if they are more severe than previous alert
This commit is contained in:
Frank Hamand 2017-03-02 15:20:02 +00:00 committed by GitHub
commit 1908f74895
3 changed files with 71 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES
View File

@ -1,6 +1,7 @@
master
------
* More severe alerts should trigger even if a less severe alert was recently sent
* Update production.env.example email settings
* Convert environment vars to boolean nicely

4
cabot/cabotapp/models.py
View File

@ -176,7 +176,9 @@ class CheckGroupMixin(models.Model):
timezone.now() - timedelta(minutes=settings.NOTIFICATION_INTERVAL)) < self.last_alert_sent:
return
elif self.overall_status in (self.CRITICAL_STATUS, self.ERROR_STATUS):
if self.last_alert_sent and (
more_important = self.old_overall_status == self.WARNING_STATUS or \
(self.old_overall_status == self.ERROR_STATUS and self.overall_status == self.CRITICAL_STATUS)
if not more_important and self.last_alert_sent and (
timezone.now() - timedelta(minutes=settings.ALERT_INTERVAL)) < self.last_alert_sent:
return
self.last_alert_sent = timezone.now()

67
cabot/cabotapp/tests/tests_basic.py
View File

@ -919,6 +919,27 @@ class TestAlerts(LocalTestCase):
def setUp(self):
super(TestAlerts, self).setUp()
self.warning_http_check = HttpStatusCheck.objects.create(
name='Http Check',
created_by=self.user,
importance=Service.WARNING_STATUS,
endpoint='http://arachnys.com',
timeout=10,
status_code='200',
text_match=None,
)
self.error_http_check = HttpStatusCheck.objects.create(
name='Http Check',
created_by=self.user,
importance=Service.ERROR_STATUS,
endpoint='http://arachnys.com',
timeout=10,
status_code='200',
text_match=None,
)
self.service.status_checks.add(self.warning_http_check, self.error_http_check)
self.critical_http_check = self.http_check
self.user.profile.hipchat_alias = "test_user_hipchat_alias"
self.user.profile.save()
@ -935,6 +956,52 @@ class TestAlerts(LocalTestCase):
self.assertEqual(fake_send_alert.call_count, 1)
fake_send_alert.assert_called_with(self.service, duty_officers=[])
def trigger_failing_check(self, check):
StatusCheckResult(
status_check=check,
time=timezone.now() - timedelta(seconds=60),
time_complete=timezone.now() - timedelta(seconds=59),
succeeded=False
).save()
check.last_run = timezone.now()
check.save()
@patch('cabot.cabotapp.models.send_alert')
def test_alert_increasing_severity(self, fake_send_alert):
self.trigger_failing_check(self.warning_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.error_http_check)
self.assertEqual(fake_send_alert.call_count, 2)
self.trigger_failing_check(self.critical_http_check)
self.assertEqual(fake_send_alert.call_count, 3)
@patch('cabot.cabotapp.models.send_alert')
def test_alert_decreasing_severity(self, fake_send_alert):
self.trigger_failing_check(self.critical_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.error_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.warning_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
@patch('cabot.cabotapp.models.send_alert')
def test_alert_alternating_severity(self, fake_send_alert):
self.trigger_failing_check(self.error_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.warning_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.error_http_check)
self.assertEqual(fake_send_alert.call_count, 1)
self.trigger_failing_check(self.critical_http_check)
self.assertEqual(fake_send_alert.call_count, 2)
def test_update_plugins(self):
# Test that disabling a plugin is detected by update_alert_plugins
plugins = update_alert_plugins()