diff --git a/cabot/cabotapp/tests/tests_basic.py b/cabot/cabotapp/tests/tests_basic.py
index b214fac..774ca32 100644
--- a/cabot/cabotapp/tests/tests_basic.py
+++ b/cabot/cabotapp/tests/tests_basic.py
@@ -8,12 +8,14 @@ from datetime import timedelta, date
 import os
 import requests
 from cabot.cabotapp.graphite import parse_metric
+from cabot.cabotapp.alert import update_alert_plugins
 from cabot.cabotapp.models import (
     GraphiteStatusCheck, JenkinsStatusCheck,
     HttpStatusCheck, ICMPStatusCheck, Service, Instance,
     StatusCheckResult, minimize_targets)
 from cabot.cabotapp.calendar import get_events
 from cabot.cabotapp.views import StatusCheckReportForm
+from django.conf import settings
 from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.core import mail
@@ -932,6 +934,16 @@ class TestAlerts(LocalTestCase):
         self.assertEqual(fake_send_alert.call_count, 1)
         fake_send_alert.assert_called_with(self.service, duty_officers=[])
 
+    def test_update_plugins(self):
+        # Test that disabling a plugin is detected by update_alert_plugins
+        plugins = update_alert_plugins()
+        plugin_count = len(plugins)
+        new_apps = [s for s in settings.INSTALLED_APPS if s not in ['cabot_alert_hipchat']]
+
+        with self.settings(INSTALLED_APPS=new_apps):
+            plugins = update_alert_plugins()
+            self.assertEqual(len(plugins), plugin_count - 1)
+
 
 class TestMinimizeTargets(LocalTestCase):
     def test_null(self):
diff --git a/cabot/cabotapp/views.py b/cabot/cabotapp/views.py
index fd96681..9a12cda 100644
--- a/cabot/cabotapp/views.py
+++ b/cabot/cabotapp/views.py
@@ -704,7 +704,9 @@ class ServiceCreateView(LoginRequiredMixin, CreateView):
     model = Service
     form_class = ServiceForm
 
-    alert.update_alert_plugins()
+    def __init__(self, *args, **kwargs):
+        alert.update_alert_plugins()
+        super(ServiceCreateView, self).__init__(*args, **kwargs)
 
     def get_success_url(self):
         return reverse('service', kwargs={'pk': self.object.id})