mirror of
https://github.com/status-im/cabot.git
synced 2025-02-24 10:28:06 +00:00
Basic alert acknowledgement mechanism
This commit is contained in:
David Buxton
parent
257bc4a819
commit
b4ab215ad2
@ -1,18 +1,33 @@
|
||||
import os
|
||||
|
||||
# Credentials for Graphite server to monitor
|
||||
GRAPHITE_API = os.environ.get('GRAPHITE_API')
|
||||
GRAPHITE_USER = os.environ.get('GRAPHITE_USER')
|
||||
GRAPHITE_PASS = os.environ.get('GRAPHITE_PASS')
|
||||
GRAPHITE_FROM = os.getenv('GRAPHITE_FROM', '-10minute')
|
||||
|
||||
# Credentials for Jenkins server to monitor
|
||||
JENKINS_API = os.environ.get('JENKINS_API')
|
||||
JENKINS_USER = os.environ.get('JENKINS_USER')
|
||||
JENKINS_PASS = os.environ.get('JENKINS_PASS')
|
||||
|
||||
# Point at a public calendar you want to use to schedule a duty rota
|
||||
CALENDAR_ICAL_URL = os.environ.get('CALENDAR_ICAL_URL')
|
||||
|
||||
# So that links back to the Cabot instance display correctly
|
||||
WWW_HTTP_HOST = os.environ.get('WWW_HTTP_HOST')
|
||||
WWW_SCHEME = os.environ.get('WWW_SCHEME', "https")
|
||||
|
||||
HTTP_USER_AGENT = os.environ.get('HTTP_USER_AGENT', 'Cabot')
|
||||
|
||||
# How often should alerts be sent for important failures?
|
||||
ALERT_INTERVAL = int(os.environ.get('ALERT_INTERVAL', 10))
|
||||
|
||||
# How often should notifications be sent for less important issues?
|
||||
NOTIFICATION_INTERVAL = int(os.environ.get('NOTIFICATION_INTERVAL', 120))
|
||||
|
||||
# How long should an acknowledgement silence alerts for?
|
||||
ACKNOWLEDGEMENT_EXPIRY = int(os.environ.get('ACKNOWLEDGEMENT_EXPIRY', 20))
|
||||
|
||||
# Default plugins are used if the user has not specified.
|
||||
CABOT_PLUGINS_ENABLED = os.environ.get('CABOT_PLUGINS_ENABLED', 'cabot_alert_hipchat,cabot_alert_twilio,cabot_alert_email')
|
||||
|
||||
@ -48,8 +48,20 @@ def send_alert(service, duty_officers=None):
|
||||
for alert in service.alerts.all():
|
||||
try:
|
||||
alert.send_alert(service, users, duty_officers)
|
||||
except Exception:
|
||||
logging.exception('Could not sent ' + alert.name + ' alert')
|
||||
except Exception as e:
|
||||
logging.exception('Could not send %s alert: %s' % (alert.name, e))
|
||||
|
||||
def send_alert_update(service, duty_officers=None):
|
||||
users = service.users_to_notify.filter(is_active=True)
|
||||
for alert in service.alerts.all():
|
||||
if hasattr(alert, 'send_alert_update'):
|
||||
try:
|
||||
alert.send_alert_update(service, users, duty_officers)
|
||||
except Exception as e:
|
||||
logger.exception('Could not send %s alert update: %s' % (alert.name, e))
|
||||
else:
|
||||
logger.warning('No send_alert_update method present for %s' % alert.name)
|
||||
|
||||
|
||||
def update_alert_plugins():
|
||||
for plugin_subclass in AlertPlugin.__subclasses__():
|
||||
|
||||
200
cabot/cabotapp/migrations/0014_auto__add_alertacknowledgement.py
Normal file
200
cabot/cabotapp/migrations/0014_auto__add_alertacknowledgement.py
Normal file
@ -0,0 +1,200 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from south.utils import datetime_utils as datetime
|
||||
from south.db import db
|
||||
from south.v2 import SchemaMigration
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(SchemaMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
# Adding model 'AlertAcknowledgement'
|
||||
db.create_table(u'cabotapp_alertacknowledgement', (
|
||||
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
|
||||
('time', self.gf('django.db.models.fields.DateTimeField')()),
|
||||
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
|
||||
('service', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['cabotapp.Service'])),
|
||||
))
|
||||
db.send_create_signal(u'cabotapp', ['AlertAcknowledgement'])
|
||||
|
||||
|
||||
def backwards(self, orm):
|
||||
# Deleting model 'AlertAcknowledgement'
|
||||
db.delete_table(u'cabotapp_alertacknowledgement')
|
||||
|
||||
|
||||
models = {
|
||||
u'auth.group': {
|
||||
'Meta': {'object_name': 'Group'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
|
||||
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
u'auth.permission': {
|
||||
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
|
||||
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
|
||||
},
|
||||
u'auth.user': {
|
||||
'Meta': {'object_name': 'User'},
|
||||
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
|
||||
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
|
||||
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
|
||||
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
|
||||
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
|
||||
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
|
||||
},
|
||||
u'cabotapp.alertacknowledgement': {
|
||||
'Meta': {'object_name': 'AlertAcknowledgement'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'service': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['cabotapp.Service']"}),
|
||||
'time': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"})
|
||||
},
|
||||
u'cabotapp.alertplugin': {
|
||||
'Meta': {'object_name': 'AlertPlugin'},
|
||||
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'polymorphic_ctype': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'polymorphic_cabotapp.alertplugin_set'", 'null': 'True', 'to': u"orm['contenttypes.ContentType']"}),
|
||||
'title': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
|
||||
},
|
||||
u'cabotapp.alertpluginuserdata': {
|
||||
'Meta': {'unique_together': "(('title', 'user'),)", 'object_name': 'AlertPluginUserData'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'polymorphic_ctype': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'polymorphic_cabotapp.alertpluginuserdata_set'", 'null': 'True', 'to': u"orm['contenttypes.ContentType']"}),
|
||||
'title': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['cabotapp.UserProfile']"})
|
||||
},
|
||||
u'cabotapp.instance': {
|
||||
'Meta': {'ordering': "['name']", 'object_name': 'Instance'},
|
||||
'address': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
|
||||
'alerts': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['cabotapp.AlertPlugin']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'alerts_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'email_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'hackpad_id': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'hipchat_alert': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'last_alert_sent': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'name': ('django.db.models.fields.TextField', [], {}),
|
||||
'old_overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'sms_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'status_checks': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['cabotapp.StatusCheck']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'telephone_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'users_to_notify': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.User']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
u'cabotapp.instancestatussnapshot': {
|
||||
'Meta': {'object_name': 'InstanceStatusSnapshot'},
|
||||
'did_send_alert': ('django.db.models.fields.IntegerField', [], {'default': 'False'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'instance': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'snapshots'", 'to': u"orm['cabotapp.Instance']"}),
|
||||
'num_checks_active': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'num_checks_failing': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'num_checks_passing': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
|
||||
},
|
||||
u'cabotapp.service': {
|
||||
'Meta': {'ordering': "['name']", 'object_name': 'Service'},
|
||||
'alerts': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['cabotapp.AlertPlugin']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'alerts_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'email_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'hackpad_id': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'hipchat_alert': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'instances': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['cabotapp.Instance']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'last_alert_sent': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'name': ('django.db.models.fields.TextField', [], {}),
|
||||
'old_overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'sms_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'status_checks': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['cabotapp.StatusCheck']", 'symmetrical': 'False', 'blank': 'True'}),
|
||||
'telephone_alert': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'url': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
|
||||
'users_to_notify': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.User']", 'symmetrical': 'False', 'blank': 'True'})
|
||||
},
|
||||
u'cabotapp.servicestatussnapshot': {
|
||||
'Meta': {'object_name': 'ServiceStatusSnapshot'},
|
||||
'did_send_alert': ('django.db.models.fields.IntegerField', [], {'default': 'False'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'num_checks_active': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'num_checks_failing': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'num_checks_passing': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'overall_status': ('django.db.models.fields.TextField', [], {'default': "'PASSING'"}),
|
||||
'service': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'snapshots'", 'to': u"orm['cabotapp.Service']"}),
|
||||
'time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'})
|
||||
},
|
||||
u'cabotapp.shift': {
|
||||
'Meta': {'object_name': 'Shift'},
|
||||
'deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'end': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'start': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'uid': ('django.db.models.fields.TextField', [], {}),
|
||||
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"})
|
||||
},
|
||||
u'cabotapp.statuscheck': {
|
||||
'Meta': {'ordering': "['name']", 'object_name': 'StatusCheck'},
|
||||
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
|
||||
'allowed_num_failures': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True'}),
|
||||
'cached_health': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
'calculated_status': ('django.db.models.fields.CharField', [], {'default': "'passing'", 'max_length': '50', 'blank': 'True'}),
|
||||
'check_type': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}),
|
||||
'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True'}),
|
||||
'debounce': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True'}),
|
||||
'endpoint': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
'expected_num_hosts': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True'}),
|
||||
'frequency': ('django.db.models.fields.IntegerField', [], {'default': '5'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'importance': ('django.db.models.fields.CharField', [], {'default': "'ERROR'", 'max_length': '30'}),
|
||||
'last_run': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
|
||||
'max_queued_build_time': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'metric': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
'name': ('django.db.models.fields.TextField', [], {}),
|
||||
'password': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'polymorphic_ctype': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'polymorphic_cabotapp.statuscheck_set'", 'null': 'True', 'to': u"orm['contenttypes.ContentType']"}),
|
||||
'status_code': ('django.db.models.fields.TextField', [], {'default': '200', 'null': 'True'}),
|
||||
'text_match': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'timeout': ('django.db.models.fields.IntegerField', [], {'default': '30', 'null': 'True'}),
|
||||
'username': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
|
||||
'value': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
'verify_ssl_certificate': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
|
||||
},
|
||||
u'cabotapp.statuscheckresult': {
|
||||
'Meta': {'ordering': "['-time_complete']", 'object_name': 'StatusCheckResult'},
|
||||
'check': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['cabotapp.StatusCheck']"}),
|
||||
'error': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'job_number': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}),
|
||||
'raw_data': ('django.db.models.fields.TextField', [], {'null': 'True'}),
|
||||
'succeeded': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
|
||||
'time_complete': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'})
|
||||
},
|
||||
u'cabotapp.userprofile': {
|
||||
'Meta': {'object_name': 'UserProfile'},
|
||||
'fallback_alert_user': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
|
||||
'hipchat_alias': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'mobile_number': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '20', 'blank': 'True'}),
|
||||
'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': u"orm['auth.User']"})
|
||||
},
|
||||
u'contenttypes.contenttype': {
|
||||
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
|
||||
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['cabotapp']
|
||||
@ -8,7 +8,13 @@ from django.contrib.auth.models import User
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
|
||||
from .jenkins import get_job_status
|
||||
from .alert import (send_alert, AlertPlugin, AlertPluginUserData, update_alert_plugins)
|
||||
from .alert import (
|
||||
send_alert,
|
||||
send_alert_update,
|
||||
AlertPlugin,
|
||||
AlertPluginUserData,
|
||||
update_alert_plugins
|
||||
)
|
||||
from .calendar import get_events
|
||||
from .graphite import parse_metric
|
||||
from .graphite import get_data
|
||||
@ -178,10 +184,29 @@ class CheckGroupMixin(models.Model):
|
||||
# We don't count "back to normal" as an alert
|
||||
self.last_alert_sent = None
|
||||
self.save()
|
||||
if self.unexpired_acknowledgement():
|
||||
send_alert_update(self, duty_officers=get_duty_officers())
|
||||
else:
|
||||
self.snapshot.did_send_alert = True
|
||||
self.snapshot.save()
|
||||
send_alert(self, duty_officers=get_duty_officers())
|
||||
|
||||
def acknowledge_alert(self, user):
|
||||
acknowledgement = AlertAcknowledgement.objects.create(
|
||||
user=user,
|
||||
time=timezone.now(),
|
||||
service=self,
|
||||
)
|
||||
|
||||
def unexpired_acknowledgement(self):
|
||||
unexpired_acknowledgements = self.alertacknowledgement_set.all().filter(
|
||||
time__gte=timezone.now()-timedelta(minutes=settings.ACKNOWLEDGEMENT_EXPIRY),
|
||||
).order_by('-time')
|
||||
try:
|
||||
return unexpired_acknowledgements[0]
|
||||
except:
|
||||
return None
|
||||
|
||||
@property
|
||||
def recent_snapshots(self):
|
||||
snapshots = self.snapshots.filter(
|
||||
@ -221,6 +246,7 @@ class CheckGroupMixin(models.Model):
|
||||
def all_failing_checks(self):
|
||||
return self.active_status_checks().exclude(calculated_status=self.CALCULATED_PASSING_STATUS)
|
||||
|
||||
|
||||
class Service(CheckGroupMixin):
|
||||
|
||||
def update_status(self):
|
||||
@ -308,6 +334,7 @@ class Instance(CheckGroupMixin):
|
||||
self.icmp_status_checks().delete()
|
||||
return super(Instance, self).delete(*args, **kwargs)
|
||||
|
||||
|
||||
class Snapshot(models.Model):
|
||||
|
||||
class Meta:
|
||||
@ -320,18 +347,21 @@ class Snapshot(models.Model):
|
||||
overall_status = models.TextField(default=Service.PASSING_STATUS)
|
||||
did_send_alert = models.IntegerField(default=False)
|
||||
|
||||
|
||||
class ServiceStatusSnapshot(Snapshot):
|
||||
service = models.ForeignKey(Service, related_name='snapshots')
|
||||
|
||||
def __unicode__(self):
|
||||
return u"%s: %s" % (self.service.name, self.overall_status)
|
||||
|
||||
|
||||
class InstanceStatusSnapshot(Snapshot):
|
||||
instance = models.ForeignKey(Instance, related_name='snapshots')
|
||||
|
||||
def __unicode__(self):
|
||||
return u"%s: %s" % (self.instance.name, self.overall_status)
|
||||
|
||||
|
||||
class StatusCheck(PolymorphicModel):
|
||||
|
||||
"""
|
||||
@ -522,6 +552,7 @@ class StatusCheck(PolymorphicModel):
|
||||
for instance in instances:
|
||||
update_instance.delay(instance.id)
|
||||
|
||||
|
||||
class ICMPStatusCheck(StatusCheck):
|
||||
|
||||
class Meta(StatusCheck.Meta):
|
||||
@ -702,6 +733,7 @@ class HttpStatusCheck(StatusCheck):
|
||||
result.succeeded = True
|
||||
return result
|
||||
|
||||
|
||||
class JenkinsStatusCheck(StatusCheck):
|
||||
|
||||
class Meta(StatusCheck.Meta):
|
||||
@ -819,6 +851,18 @@ class StatusCheckResult(models.Model):
|
||||
return super(StatusCheckResult, self).save(*args, **kwargs)
|
||||
|
||||
|
||||
class AlertAcknowledgement(models.Model):
|
||||
|
||||
time = models.DateTimeField()
|
||||
user = models.ForeignKey(User)
|
||||
service = models.ForeignKey(Service)
|
||||
|
||||
def unexpired(self):
|
||||
return self.expires() > timezone.now()
|
||||
|
||||
def expires(self):
|
||||
return self.time + timedelta(minutes=settings.ACKNOWLEDGEMENT_EXPIRY)
|
||||
|
||||
class UserProfile(models.Model):
|
||||
user = models.OneToOneField(User, related_name='profile')
|
||||
|
||||
@ -845,6 +889,7 @@ class UserProfile(models.Model):
|
||||
hipchat_alias = models.CharField(max_length=50, blank=True, default='')
|
||||
fallback_alert_user = models.BooleanField(default=False)
|
||||
|
||||
|
||||
class Shift(models.Model):
|
||||
start = models.DateTimeField()
|
||||
end = models.DateTimeField()
|
||||
|
||||
@ -170,7 +170,6 @@ def throws_timeout(*args, **kwargs):
|
||||
|
||||
class TestCheckRun(LocalTestCase):
|
||||
|
||||
|
||||
def test_calculate_service_status(self):
|
||||
self.assertEqual(self.graphite_check.calculated_status,
|
||||
Service.CALCULATED_PASSING_STATUS)
|
||||
@ -208,6 +207,31 @@ class TestCheckRun(LocalTestCase):
|
||||
self.service.update_status()
|
||||
self.assertEqual(self.service.overall_status, Service.PASSING_STATUS)
|
||||
|
||||
@patch('cabot.cabotapp.models.send_alert')
|
||||
@patch('cabot.cabotapp.models.send_alert_update')
|
||||
def test_alert_acknowledgement(self, fake_send_alert_update, fake_send_alert):
|
||||
self.assertEqual(self.service.overall_status, Service.PASSING_STATUS)
|
||||
self.most_recent_result.succeeded = False
|
||||
self.most_recent_result.save()
|
||||
self.graphite_check.last_run = timezone.now()
|
||||
self.graphite_check.save()
|
||||
self.assertEqual(self.graphite_check.calculated_status,
|
||||
Service.CALCULATED_FAILING_STATUS)
|
||||
self.service.update_status()
|
||||
fake_send_alert.assert_called_with(self.service, duty_officers=[])
|
||||
|
||||
fake_send_alert.reset_mock()
|
||||
self.service.last_alert_sent = timezone.now() - timedelta(minutes=30)
|
||||
self.service.update_status()
|
||||
fake_send_alert.assert_called_with(self.service, duty_officers=[])
|
||||
|
||||
fake_send_alert.reset_mock()
|
||||
self.service.acknowledge_alert(user=self.user)
|
||||
self.service.last_alert_sent = timezone.now() - timedelta(minutes=30)
|
||||
self.service.update_status()
|
||||
self.assertEqual(self.service.unexpired_acknowledgement().user, self.user)
|
||||
fake_send_alert_update.assert_called_with(self.service, duty_officers=[])
|
||||
|
||||
@patch('cabot.cabotapp.graphite.requests.get', fake_graphite_response)
|
||||
def test_graphite_run(self):
|
||||
checkresults = self.graphite_check.statuscheckresult_set.all()
|
||||
|
||||
@ -51,14 +51,12 @@ def subscriptions(request):
|
||||
})
|
||||
return HttpResponse(t.render(c))
|
||||
|
||||
|
||||
@login_required
|
||||
def run_status_check(request, pk):
|
||||
"""Runs a specific check"""
|
||||
_run_status_check(check_or_id=pk)
|
||||
return HttpResponseRedirect(reverse('check', kwargs={'pk': pk}))
|
||||
|
||||
|
||||
def duplicate_icmp_check(request, pk):
|
||||
pc = StatusCheck.objects.get(pk=pk)
|
||||
npk = pc.duplicate()
|
||||
@ -666,6 +664,14 @@ class InstanceCreateView(LoginRequiredMixin, CreateView):
|
||||
|
||||
return initial
|
||||
|
||||
|
||||
@login_required
|
||||
def acknowledge_alert(request, service_id):
|
||||
service = Service.objects.get(pk=service_id)
|
||||
service.acknowledge_alert(user=request.user)
|
||||
return HttpResponseRedirect(reverse('service', kwargs={'pk': pk}))
|
||||
|
||||
|
||||
class ServiceCreateView(LoginRequiredMixin, CreateView):
|
||||
model = Service
|
||||
form_class = ServiceForm
|
||||
@ -674,6 +680,7 @@ class ServiceCreateView(LoginRequiredMixin, CreateView):
|
||||
def get_success_url(self):
|
||||
return reverse('service', kwargs={'pk': self.object.id})
|
||||
|
||||
|
||||
class InstanceUpdateView(LoginRequiredMixin, UpdateView):
|
||||
model = Instance
|
||||
form_class = InstanceForm
|
||||
@ -681,6 +688,7 @@ class InstanceUpdateView(LoginRequiredMixin, UpdateView):
|
||||
def get_success_url(self):
|
||||
return reverse('instance', kwargs={'pk': self.object.id})
|
||||
|
||||
|
||||
class ServiceUpdateView(LoginRequiredMixin, UpdateView):
|
||||
model = Service
|
||||
form_class = ServiceForm
|
||||
|
||||
@ -11,6 +11,18 @@
|
||||
<div class="col-xs-4 text-right"><h2><span class="label label-{% if service.overall_status == service.PASSING_STATUS %}success{% else %}danger{% endif %}">{{ service.overall_status|lower|capfirst }}</span> <span class="label label-{% if service.alerts_enabled %}success{% else %}warning{% endif %}">{% if service.alerts_enabled %}Alerts enabled{%else %}Alerts disabled{% endif %}</span></h2></div>
|
||||
<div class="col-xs-2 text-right"><h2><a href="{% url "update-service" service.id %}"><i class="glyphicon glyphicon-edit"></i></a></h2></div>
|
||||
</div>
|
||||
{% if service.overall_status != service.PASSING_STATUS %}
|
||||
{% if service.unexpired_acknowledgement %}
|
||||
<div class="col-xs-12 alert alert-primary">
|
||||
<i class="glyphicon glyphicon-pause"></i> {{ service.unexpired_acknowledgement.user.email }} acknowledged an alert against this service at {{ service.unexpired_acknowledgement.time }} - alerts are currently paused. The acknowledgement will expire at {{ service.unexpired_acknowledgement.expires }}.
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="col-xs-12 alert alert-primary">
|
||||
<a class="btn btn-primary" href="{% url 'acknowledge-alert' pk=service.id %}"><i class=""></i>Acknowledge alert</a>
|
||||
By acknowledging this failure you will pause alerts temporarily.
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
<hr>
|
||||
<div class="row">
|
||||
|
||||
@ -2,7 +2,8 @@ from django.conf.urls import patterns, include, url
|
||||
from django.conf import settings
|
||||
from cabot.cabotapp.views import (
|
||||
run_status_check, graphite_api_data, checks_run_recently,
|
||||
duplicate_icmp_check, duplicate_graphite_check, duplicate_http_check, duplicate_jenkins_check, duplicate_instance,
|
||||
duplicate_icmp_check, duplicate_graphite_check, duplicate_http_check, duplicate_jenkins_check,
|
||||
duplicate_instance, acknowledge_alert,
|
||||
GraphiteCheckCreateView, GraphiteCheckUpdateView,
|
||||
HttpCheckCreateView, HttpCheckUpdateView,
|
||||
ICMPCheckCreateView, ICMPCheckUpdateView,
|
||||
@ -56,6 +57,8 @@ urlpatterns = patterns('',
|
||||
), name='delete-service'),
|
||||
url(r'^service/(?P<pk>\d+)/',
|
||||
view=ServiceDetailView.as_view(), name='service'),
|
||||
url(r'^service/acknowledge_alert/(?P<pk>\d+)/',
|
||||
view=acknowledge_alert, name='acknowledge-alert'),
|
||||
|
||||
url(r'^instances/', view=InstanceListView.as_view(),
|
||||
name='instances'),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user