status-im/cabot
2
0
Fork 0
mirror of https://github.com/status-im/cabot.git synced 2025-02-24 02:18:08 +00:00
Code Issues Projects Releases Wiki Activity

Allow multiple Jenkins backends for JenkinsStatusCheck

Browse Source 
We added a JenkinsConfig model to store configs (atm only editable in
django admin - to be moved to some sort of plugins options page)

If no JenkinsConfig are available, one will be created from the current
environment variables (JENKINS_URL etc)

Refactor JenkinsStatusCheck to have its own table instead of being a
proxy model. This is because we're moving to a plugin architecture
for status checks so we can't keep adding every new field to the base
model.
This commit is contained in:
Frank Hamand 2017-08-21 18:03:35 +01:00
parent e0b36009d2
commit b03c4589c4
12 changed files with 257 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cabot/cabotapp/admin.py
View File

@ -1,16 +1,33 @@
from django.contrib import admin
from .models import (UserProfile, Service, Shift,
ServiceStatusSnapshot, StatusCheck, StatusCheckResult,
Instance, AlertAcknowledgement)
from .alert import AlertPluginUserData, AlertPlugin
from polymorphic.admin import (PolymorphicChildModelAdmin,
PolymorphicParentModelAdmin)
from .alert import AlertPlugin, AlertPluginUserData
from .models import (AlertAcknowledgement, Instance, JenkinsConfig, Service,
ServiceStatusSnapshot, Shift, StatusCheck,
StatusCheckResult, UserProfile)
class StatusCheckAdmin(PolymorphicParentModelAdmin):
base_model = StatusCheck
child_models = StatusCheck.__subclasses__()
class ChildStatusCheckAdmin(PolymorphicChildModelAdmin):
base_model = StatusCheck
for child_status_check in StatusCheck.__subclasses__():
admin.site.register(child_status_check, ChildStatusCheckAdmin)
admin.site.register(UserProfile)
admin.site.register(Shift)
admin.site.register(Service)
admin.site.register(ServiceStatusSnapshot)
admin.site.register(StatusCheck)
admin.site.register(StatusCheck, StatusCheckAdmin)
admin.site.register(StatusCheckResult)
admin.site.register(Instance)
admin.site.register(AlertPlugin)
admin.site.register(AlertPluginUserData)
admin.site.register(AlertAcknowledgement)
admin.site.register(JenkinsConfig)

3
cabot/cabotapp/apps.py
View File

@ -4,8 +4,9 @@ from django.db.models.signals import post_migrate
def post_migrate_callback(**kwargs):
from cabot.cabotapp.alert import update_alert_plugins
from cabot.cabotapp.models import create_default_jenkins_config
update_alert_plugins()
create_default_jenkins_config()
class CabotappConfig(AppConfig):
name = 'cabot.cabotapp'

12
cabot/cabotapp/jenkins.py
View File

@ -11,23 +11,23 @@ logger = get_task_logger(__name__)
JENKINS_CLIENT = None
def _get_jenkins_client():
def _get_jenkins_client(jenkins_config):
global JENKINS_CLIENT
if JENKINS_CLIENT is None:
JENKINS_CLIENT = Jenkins(settings.JENKINS_API,
username=settings.JENKINS_USER,
password=settings.JENKINS_PASS,
JENKINS_CLIENT = Jenkins(jenkins_config.jenkins_api,
username=jenkins_config.jenkins_user,
password=jenkins_config.jenkins_pass,
lazy=True)
return JENKINS_CLIENT
def get_job_status(jobname):
def get_job_status(jenkins_config, jobname):
ret = {
'active': None,
'succeeded': None,
'job_number': None,
'blocked_build_time': None,
}
client = _get_jenkins_client()
client = _get_jenkins_client(jenkins_config)
try:
job = client.get_job(jobname)
last_build = job.get_last_build()

94
cabot/cabotapp/migrations/0005_auto_20170818_1202.py Normal file
View File

@ -0,0 +1,94 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-08-18 12:02
from __future__ import unicode_literals
import os
import django.db.models.deletion
from django.db import migrations, models
def move_old_jenkins_checks(apps, schema_editor):
db_alias = schema_editor.connection.alias
JenkinsStatusCheck = apps.get_model("cabotapp", "JenkinsStatusCheck")
JenkinsCheck = apps.get_model("cabotapp", "JenkinsCheck")
JenkinsConfig = apps.get_model("cabotapp", "JenkinsConfig")
if not JenkinsConfig.objects.exists():
JenkinsConfig.objects.create(
name="Default Jenkins",
jenkins_api=os.environ.get("JENKINS_API"),
jenkins_user=os.environ.get("JENKINS_USER"),
jenkins_pass=os.environ.get("JENKINS_PASS"),
)
default_config = JenkinsConfig.objects.first()
for old_check in JenkinsStatusCheck.objects.all():
new_check = JenkinsCheck(
active=old_check.active,
allowed_num_failures=old_check.allowed_num_failures,
cached_health=old_check.cached_health,
calculated_status=old_check.calculated_status,
check_type=old_check.check_type,
created_by_id=old_check.created_by_id,
debounce=old_check.debounce,
endpoint=old_check.endpoint,
expected_num_hosts=old_check.expected_num_hosts,
frequency=old_check.frequency,
importance=old_check.importance,
last_run=old_check.last_run,
max_queued_build_time=old_check.max_queued_build_time,
metric=old_check.metric,
name=old_check.name,
password=old_check.password,
status_code=old_check.status_code,
text_match=old_check.text_match,
timeout=old_check.timeout,
username=old_check.username,
value=old_check.value,
jenkins_config=default_config,
# For some reason this isn't handled automatically...
# The model is renamed in the next migration so the ctype
# id stays consistent.
polymorphic_ctype_id=old_check.polymorphic_ctype_id
)
new_check.save(using=db_alias)
old_check.delete(using=db_alias)
class Migration(migrations.Migration):
dependencies = [
('cabotapp', '0004_auto_20170802_1327'),
]
operations = [
migrations.CreateModel(
name='JenkinsCheck',
fields=[
('statuscheck_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='cabotapp.StatusCheck')),
],
options={
'abstract': False,
},
bases=('cabotapp.statuscheck',),
),
migrations.CreateModel(
name='JenkinsConfig',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=30)),
('jenkins_api', models.CharField(max_length=2000)),
('jenkins_user', models.CharField(max_length=2000)),
('jenkins_pass', models.CharField(max_length=2000)),
],
),
migrations.AddField(
model_name='jenkinscheck',
name='jenkins_config',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cabotapp.JenkinsConfig'),
),
migrations.RunPython(move_old_jenkins_checks)
]

22
cabot/cabotapp/migrations/0006_auto_20170821_1000.py Normal file
View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-08-21 10:00
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('cabotapp', '0005_auto_20170818_1202'),
]
operations = [
migrations.DeleteModel(
name='JenkinsStatusCheck',
),
migrations.RenameModel(
old_name='JenkinsCheck',
new_name='JenkinsStatusCheck',
)
]

2
cabot/cabotapp/models/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from .base import *
from .jenkins_check_plugin import *

75
cabot/cabotapp/models.py → cabot/cabotapp/models/base.py
View File

@ -6,6 +6,7 @@ import time
from datetime import timedelta
import requests
from celery.exceptions import SoftTimeLimitExceeded
from celery.utils.log import get_task_logger
from django.conf import settings
@ -13,19 +14,12 @@ from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import post_save
from django.utils import timezone
from polymorphic.models import PolymorphicModel
from .alert import (
send_alert,
send_alert_update,
AlertPluginUserData
)
from .calendar import get_events
from .graphite import parse_metric
from .jenkins import get_job_status
from .tasks import update_service, update_instance
from ..alert import AlertPluginUserData, send_alert, send_alert_update
from ..calendar import get_events
from ..graphite import parse_metric
from ..tasks import update_instance, update_service
RAW_DATA_LIMIT = 5000
@ -781,65 +775,6 @@ class HttpStatusCheck(StatusCheck):
return result
class JenkinsStatusCheck(StatusCheck):
class Meta(StatusCheck.Meta):
proxy = True
@property
def check_category(self):
return "Jenkins check"
@property
def failing_short_status(self):
return 'Job failing on Jenkins'
def _run(self):
result = StatusCheckResult(status_check=self)
try:
status = get_job_status(self.name)
active = status['active']
result.job_number = status['job_number']
if status['status_code'] == 404:
result.error = u'Job %s not found on Jenkins' % self.name
result.succeeded = False
return result
elif status['status_code'] > 400:
# Will fall through to next block
raise Exception(u'returned %s' % status['status_code'])
except Exception as e:
# If something else goes wrong, we will *not* fail - otherwise
# a lot of services seem to fail all at once.
# Ugly to do it here but...
result.error = u'Error fetching from Jenkins - %s' % e.message
result.succeeded = True
return result
if not active:
# We will fail if the job has been disabled
result.error = u'Job "%s" disabled on Jenkins' % self.name
result.succeeded = False
else:
if self.max_queued_build_time and status['blocked_build_time']:
if status['blocked_build_time'] > self.max_queued_build_time * 60:
result.succeeded = False
result.error = u'Job "%s" has blocked build waiting for %ss (> %sm)' % (
self.name,
int(status['blocked_build_time']),
self.max_queued_build_time,
)
else:
result.succeeded = status['succeeded']
else:
result.succeeded = status['succeeded']
if not status['succeeded']:
if result.error:
result.error += u'; Job "%s" failing on Jenkins' % self.name
else:
result.error = u'Job "%s" failing on Jenkins' % self.name
result.raw_data = status
return result
class StatusCheckResult(models.Model):
"""
We use the same StatusCheckResult model for all check types,

84
cabot/cabotapp/models/jenkins_check_plugin.py Normal file
View File

@ -0,0 +1,84 @@
import os
from django.db import models
from ..jenkins import get_job_status
from .base import StatusCheck, StatusCheckResult
class JenkinsStatusCheck(StatusCheck):
jenkins_config = models.ForeignKey('JenkinsConfig')
@property
def check_category(self):
return "Jenkins check"
@property
def failing_short_status(self):
return 'Job failing on Jenkins'
def _run(self):
result = StatusCheckResult(status_check=self)
try:
status = get_job_status(self.jenkins_config, self.name)
active = status['active']
result.job_number = status['job_number']
if status['status_code'] == 404:
result.error = u'Job %s not found on Jenkins' % self.name
result.succeeded = False
return result
elif status['status_code'] > 400:
# Will fall through to next block
raise Exception(u'returned %s' % status['status_code'])
except Exception as e:
# If something else goes wrong, we will *not* fail - otherwise
# a lot of services seem to fail all at once.
# Ugly to do it here but...
result.error = u'Error fetching from Jenkins - %s' % e.message
result.succeeded = True
return result
if not active:
# We will fail if the job has been disabled
result.error = u'Job "%s" disabled on Jenkins' % self.name
result.succeeded = False
else:
if self.max_queued_build_time and status['blocked_build_time']:
if status['blocked_build_time'] > self.max_queued_build_time * 60:
result.succeeded = False
result.error = u'Job "%s" has blocked build waiting for %ss (> %sm)' % (
self.name,
int(status['blocked_build_time']),
self.max_queued_build_time,
)
else:
result.succeeded = status['succeeded']
else:
result.succeeded = status['succeeded']
if not status['succeeded']:
if result.error:
result.error += u'; Job "%s" failing on Jenkins' % self.name
else:
result.error = u'Job "%s" failing on Jenkins' % self.name
result.raw_data = status
return result
class JenkinsConfig(models.Model):
name = models.CharField(max_length=30, blank=False)
jenkins_api = models.CharField(max_length=2000, blank=False)
jenkins_user = models.CharField(max_length=2000, blank=False)
jenkins_pass = models.CharField(max_length=2000, blank=False)
def __str__(self):
return self.name
def create_default_jenkins_config():
if not JenkinsConfig.objects.exists():
JenkinsConfig.objects.create(
name="Default Jenkins",
jenkins_api=os.environ.get("JENKINS_API"),
jenkins_user=os.environ.get("JENKINS_USER"),
jenkins_pass=os.environ.get("JENKINS_PASS"),
)

15
cabot/cabotapp/tests/tests_basic.py
View File

@ -10,10 +10,10 @@ import requests
from cabot.cabotapp.graphite import parse_metric
from cabot.cabotapp.alert import update_alert_plugins, AlertPlugin
from cabot.cabotapp.models import (
GraphiteStatusCheck, JenkinsStatusCheck,
GraphiteStatusCheck, JenkinsStatusCheck, JenkinsConfig,
HttpStatusCheck, ICMPStatusCheck, Service, Instance,
StatusCheckResult, minimize_targets, ServiceStatusSnapshot,
add_custom_check_plugins)
add_custom_check_plugins, create_default_jenkins_config)
from cabot.cabotapp.calendar import get_events
from cabot.cabotapp.views import StatusCheckReportForm
from cabot.cabotapp import tasks
@ -76,11 +76,13 @@ class LocalTestCase(APITestCase):
created_by=self.user,
importance=Service.ERROR_STATUS,
)
create_default_jenkins_config()
self.jenkins_check = JenkinsStatusCheck.objects.create(
name='Jenkins Check',
created_by=self.user,
importance=Service.ERROR_STATUS,
max_queued_build_time=10,
jenkins_config = JenkinsConfig.objects.first()
)
self.http_check = HttpStatusCheck.objects.create(
name='Http Check',
@ -394,7 +396,7 @@ class TestCheckRun(LocalTestCase):
self.assertTrue(self.graphite_check.last_result().succeeded)
self.assertGreater(list(checkresults)[-1].took, 0.0)
@patch('cabot.cabotapp.models.get_job_status')
@patch('cabot.cabotapp.models.jenkins_check_plugin.get_job_status')
def test_jenkins_run(self, mock_get_job_status):
mock_get_job_status.return_value = fake_jenkins_response()
checkresults = self.jenkins_check.statuscheckresult_set.all()
@ -406,7 +408,7 @@ class TestCheckRun(LocalTestCase):
self.assertEqual(len(checkresults), 1)
self.assertFalse(self.jenkins_check.last_result().succeeded)
@patch('cabot.cabotapp.models.get_job_status')
@patch('cabot.cabotapp.models.jenkins_check_plugin.get_job_status')
def test_jenkins_blocked_build(self, mock_get_job_status):
mock_get_job_status.return_value = jenkins_blocked_response()
checkresults = self.jenkins_check.statuscheckresult_set.all()
@ -764,6 +766,7 @@ class TestAPI(LocalTestCase):
'max_queued_build_time': 10,
'id': self.jenkins_check.id,
'calculated_status': u'passing',
'jenkins_config': JenkinsConfig.objects.first().id,
},
],
'icmpstatuscheck': [
@ -850,6 +853,7 @@ class TestAPI(LocalTestCase):
'max_queued_build_time': 37,
'id': self.jenkins_check.id,
'calculated_status': u'passing',
'jenkins_config': JenkinsConfig.objects.first().id,
},
],
'icmpstatuscheck': [
@ -930,16 +934,19 @@ class TestAPIFiltering(LocalTestCase):
name='Filter test 1',
debounce=True,
importance=Service.CRITICAL_STATUS,
jenkins_config=JenkinsConfig.objects.first()
)
JenkinsStatusCheck.objects.create(
name='Filter test 2',
debounce=True,
importance=Service.WARNING_STATUS,
jenkins_config=JenkinsConfig.objects.first()
)
JenkinsStatusCheck.objects.create(
name='Filter test 3',
debounce=False,
importance=Service.CRITICAL_STATUS,
jenkins_config=JenkinsConfig.objects.first()
)
GraphiteStatusCheck.objects.create(

11
cabot/cabotapp/tests/tests_jenkins.py
View File

@ -4,6 +4,7 @@ import unittest
from freezegun import freeze_time
from mock import patch, create_autospec
from cabot.cabotapp import jenkins
from cabot.cabotapp.models import JenkinsConfig
from django.utils import timezone
from datetime import timedelta
import jenkinsapi
@ -22,6 +23,8 @@ class TestGetStatus(unittest.TestCase):
self.mock_client = create_autospec(jenkinsapi.jenkins.Jenkins)
self.mock_client.get_job.return_value = self.mock_job
self.mock_config = create_autospec(JenkinsConfig)
@patch("cabot.cabotapp.jenkins._get_jenkins_client")
def test_job_passing(self, mock_jenkins):
mock_jenkins.return_value = self.mock_client
@ -29,7 +32,7 @@ class TestGetStatus(unittest.TestCase):
self.mock_build.is_good.return_value = True
self.mock_job.is_queued.return_value = False
status = jenkins.get_job_status('foo')
status = jenkins.get_job_status(self.mock_config, 'foo')
expected = {
'active': True,
@ -47,7 +50,7 @@ class TestGetStatus(unittest.TestCase):
self.mock_build.is_good.return_value = False
self.mock_job.is_queued.return_value = False
status = jenkins.get_job_status('foo')
status = jenkins.get_job_status(self.mock_config, 'foo')
expected = {
'active': True,
@ -71,7 +74,7 @@ class TestGetStatus(unittest.TestCase):
}
}
with freeze_time(timezone.now() + timedelta(minutes=10)):
status = jenkins.get_job_status('foo')
status = jenkins.get_job_status(self.mock_config, 'foo')
expected = {
'active': True,
@ -87,7 +90,7 @@ class TestGetStatus(unittest.TestCase):
self.mock_client.get_job.side_effect = UnknownJob()
mock_jenkins.return_value = self.mock_client
status = jenkins.get_job_status('unknown-job')
status = jenkins.get_job_status(self.mock_config, 'unknown-job')
expected = {
'active': None,

1
cabot/cabotapp/views.py
View File

@ -252,6 +252,7 @@ class JenkinsStatusCheckForm(StatusCheckForm):
'importance',
'debounce',
'max_queued_build_time',
'jenkins_config',
)
widgets = dict(**base_widgets)

1
cabot/rest_urls.py
View File

@ -112,6 +112,7 @@ router.register(r'jenkins_checks', create_viewset(
arg_model=models.JenkinsStatusCheck,
arg_fields=status_check_fields + (
'max_queued_build_time',
'jenkins_config',
),
))