From 2f79bc5a23edb9abddf05c87f21571b2a023777d Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 9 Feb 2017 18:32:00 +0000 Subject: [PATCH 1/8] Switch to using alpine linux for docker container --- Dockerfile | 21 +++++++++++---------- requirements-dev.txt | 2 +- requirements.txt | 1 - 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3a16a2..becbb7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7 +FROM node:4-alpine ENV PYTHONUNBUFFERED 1 @@ -6,31 +6,32 @@ RUN mkdir /code WORKDIR /code -RUN apt-get update && apt-get install -y \ +RUN apk add --no-cache \ python-dev \ - libsasl2-dev \ - libldap2-dev \ - libpq-dev \ - npm + py-pip \ + postgresql-dev \ + gcc \ + musl-dev \ + libffi-dev \ + openldap-dev \ + bash RUN npm install -g \ --registry http://registry.npmjs.org/ \ coffee-script \ less@1.3 -RUN ln -s `which nodejs` /usr/bin/node - RUN pip install --upgrade pip COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + COPY requirements-dev.txt ./ RUN pip install --no-cache-dir -r requirements-dev.txt COPY requirements-plugins.txt ./ RUN pip install --no-cache-dir -r requirements-plugins.txt -RUN pip install ipdb - ADD . /code/ ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d6ccab6..576621c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ --r requirements.txt coverage==4.2 django_coverage_plugin==1.3.1 mock==1.0.1 +ipdb diff --git a/requirements.txt b/requirements.txt index 103635d..f21ecdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ django-filter==0.13 django-jsonify==0.3.0 django-mptt==0.6.0 django-polymorphic==0.7.2 -django-redis==1.4.5 django-smtp-ssl==1.0 djangorestframework==2.4.8 gunicorn==18.0 From ec5f6825e25aaf8e93c172d0cade0688bb2dddeb Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 9 Feb 2017 21:37:07 +0000 Subject: [PATCH 2/8] Add default.env and test.env and multiple docker-compose files to override settings --- .travis.yml | 2 +- cabot/celeryconfig.py | 3 +- conf/default.env | 74 +++++++++++++++++++++++++++++++++++ conf/test.env | 7 ++++ docker-compose-base.yml | 10 +++++ docker-compose-production.yml | 35 +++++++++++++++++ docker-compose-test.yml | 9 +++++ docker-compose.yml | 48 +++++++++++++---------- docker-entrypoint.sh | 14 ++++++- requirements.txt | 3 +- 10 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 conf/default.env create mode 100644 conf/test.env create mode 100644 docker-compose-base.yml create mode 100644 docker-compose-production.yml create mode 100644 docker-compose-test.yml diff --git a/.travis.yml b/.travis.yml index 5ad3f93..35677d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_script: script: - tox - - docker-compose run --rm web bash bin/test_with_coverage + - docker-compose run --rm -e SKIP_INIT=1 -e DATABASE_URL='sqlite://:memory:' web bash bin/test_with_coverage -v2 - docker-compose run --rm -e CABOT_SUPERUSER_USERNAME='admin' -e CABOT_SUPERUSER_PASSWORD='pass' web true after_success: diff --git a/cabot/celeryconfig.py b/cabot/celeryconfig.py index 9d7f328..4b715a0 100644 --- a/cabot/celeryconfig.py +++ b/cabot/celeryconfig.py @@ -4,6 +4,7 @@ from datetime import timedelta BROKER_URL = os.environ['CELERY_BROKER_URL'] # Set environment variable if you want to run tests without a redis instance CELERY_ALWAYS_EAGER = os.environ.get('CELERY_ALWAYS_EAGER', False) +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', None) CELERY_IMPORTS = ('cabot.cabotapp.tasks', ) CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler" CELERY_TASK_SERIALIZER = "json" @@ -22,7 +23,7 @@ CELERYBEAT_SCHEDULE = { }, 'clean-db': { 'task': 'cabot.cabotapp.tasks.clean_db', - 'schedule': timedelta(seconds=60*60*24), + 'schedule': timedelta(seconds=60 * 60 * 24), }, } diff --git a/conf/default.env b/conf/default.env new file mode 100644 index 0000000..fd7eed8 --- /dev/null +++ b/conf/default.env @@ -0,0 +1,74 @@ +# Plugins to be loaded at launch +CABOT_PLUGINS_ENABLED=cabot_alert_hipchat==1.8.3,cabot_alert_twilio==1.2.0,cabot_alert_email==1.4.3 + +DEBUG=t +DATABASE_URL=postgres://postgres@db:5432/postgres +DJANGO_SETTINGS_MODULE=cabot.settings +HIPCHAT_URL=https://api.hipchat.com/v1/rooms/message +LOG_FILE=/dev/null +PORT=5001 + +# You shouldn't need to change anything above this line + +# Base path to include before generated URLs. If not defined, uses `/` +# URL_PREFIX=/ + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +TIME_ZONE=Etc/UTC + +# URL of calendar to synchronise rota with +CALENDAR_ICAL_URL=http://www.google.com/calendar/ical/example.ics + +# Django settings +CELERY_BROKER_URL=redis://redis:6379/1 + +# From parameter for the graphite request. If not defined, by default take -10 minutes +# GRAPHITE_FROM=-10minute + +# User-Agent string used for HTTP checks +HTTP_USER_AGENT=Cabot + +# Used for pointing links back in alerts etc. +WWW_HTTP_HOST=localhost +WWW_SCHEME=http + +# OPTIONAL SETTINGS +# +# # Django admin email +# ADMIN_EMAIL=you@example.com +# CABOT_FROM_EMAIL=cabot@example.com +# +# DJANGO_SECRET_KEY= +# +# Hostname of your Graphite server instance +GRAPHITE_API=http://graphite.example.com/ +GRAPHITE_USER=username +GRAPHITE_PASS=password + +# Hipchat integration +HIPCHAT_ALERT_ROOM=48052 +HIPCHAT_API_KEY=your_hipchat_api_key + +# Jenkins integration +JENKINS_API=https://jenkins.example.com/ +JENKINS_USER=username +JENKINS_PASS=password + +# SMTP settings +SES_HOST=email-smtp.us-east-1.amazonaws.com +SES_USER=username +SES_PASS=password +SES_PORT=465 + +# Twilio integration for SMS and telephone alerts +TWILIO_ACCOUNT_SID=your_account_sid +TWILIO_AUTH_TOKEN=your_auth_token +TWILIO_OUTGOING_NUMBER=+14155551234 + +# Use for LDAP authentication +AUTH_LDAP=true +AUTH_LDAP_SERVER_URI=ldap://ldap.example.com +AUTH_LDAP_BIND_DN="cn=Manager,dc=example,dc=com" +AUTH_LDAP_BIND_PASSWORD="" +AUTH_LDAP_USER_SEARCH="ou=People,dc=example,dc=com" diff --git a/conf/test.env b/conf/test.env new file mode 100644 index 0000000..5312149 --- /dev/null +++ b/conf/test.env @@ -0,0 +1,7 @@ +DATABASE_URL=sqlite://:memory: + +CELERY_BROKER_URL=sqla+sqlite://:memory: +CELERY_RESULT_BACKEND=db+sqlite://:memory: +CELERY_ALWAYS_EAGER=true + +SKIP_INIT=true diff --git a/docker-compose-base.yml b/docker-compose-base.yml new file mode 100644 index 0000000..caffa6a --- /dev/null +++ b/docker-compose-base.yml @@ -0,0 +1,10 @@ +version: '2' +services: + base: + build: . + image: cabot:web + command: "false" + volumes: + - .:/code + env_file: + - conf/default.env diff --git a/docker-compose-production.yml b/docker-compose-production.yml new file mode 100644 index 0000000..83b86dc --- /dev/null +++ b/docker-compose-production.yml @@ -0,0 +1,35 @@ +version: '2' +services: + base: + extends: + file: docker-compose-base.yml + service: base + env_file: + - conf/production.env + + web: + extends: base + command: gunicorn cabot.wsgi:application --config gunicorn.conf + links: + - redis + - db + + worker: + extends: base + command: python manage.py celery worker -A cabot --loglevel=DEBUG --concurrency=16 -Ofair + links: + - redis + - db + + beat: + extends: base + command: python manage.py celery beat -A cabot --loglevel=DEBUG + links: + - redis + - db + + redis: + image: redis + + db: + image: postgres diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 0000000..a82c05b --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,9 @@ +version: '2' +services: + test: + extends: + file: docker-compose-base.yml + service: base + command: python manage.py test -v2 + env_file: + - conf/test.env diff --git a/docker-compose.yml b/docker-compose.yml index 14149ad..4d40362 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,48 @@ version: '2' services: web: + extends: + file: docker-compose-base.yml + service: base env_file: - - conf/development.env - build: . - image: cabot:web + - conf/development.env + environment: + - CABOT_SUPERUSER_USERNAME=admin + - CABOT_SUPERUSER_PASSWORD=pass command: python manage.py runserver 0.0.0.0:5001 ports: - - "5001:5001" - volumes: - - .:/code + - "5000:5000" links: - - redis - - db + - redis + - db worker: + extends: + file: docker-compose-base.yml + service: base env_file: - - conf/development.env - image: cabot:web + - conf/development.env command: python manage.py celery worker -A cabot --loglevel=DEBUG --concurrency=16 -Ofair - volumes: - - .:/code + environment: + - SKIP_INIT=1 + - WAIT_FOR_MIGRATIONS=1 links: - - redis - - db + - redis + - db beat: + extends: + file: docker-compose-base.yml + service: base env_file: - - conf/development.env - image: cabot:web + - conf/development.env command: python manage.py celery beat -A cabot --loglevel=DEBUG - volumes: - - .:/code + environment: + - SKIP_INIT=1 + - WAIT_FOR_MIGRATIONS=1 links: - - redis - - db + - redis + - db redis: image: redis diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 15f02f9..1b1b9cd 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,6 +1,5 @@ #!/bin/bash set -e -set -o allexport function wait_for_broker {( set +e @@ -20,6 +19,15 @@ function wait_for_database {( done )} +function wait_for_migrations {( + set +e + for try in {1..60} ; do + python manage.py migrate --list | grep "\[ \]" &> /dev/null || break + echo "Waiting for database migrations to be run..." + sleep 1 + done +)} + wait_for_broker wait_for_database @@ -28,4 +36,8 @@ if [ -z "$SKIP_INIT" ]; then /code/bin/build-app fi +if [ -n "$WAIT_FOR_MIGRATIONS" ]; then + wait_for_migrations +fi + exec "$@" diff --git a/requirements.txt b/requirements.txt index f21ecdb..bcca038 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ django-mptt==0.6.0 django-polymorphic==0.7.2 django-smtp-ssl==1.0 djangorestframework==2.4.8 -gunicorn==18.0 +gunicorn==19.6.0 gevent==1.0.1 httplib2==0.7.7 icalendar==3.2 @@ -30,3 +30,4 @@ twilio==3.4.1 wsgiref==0.1.2 python-dateutil==2.1 django-auth-ldap==1.2.6 +sqlalchemy==1.1.5 From bc1b8b79d7e15b92903355e88c793d9d0bb8bc44 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Mon, 13 Feb 2017 13:55:09 +0000 Subject: [PATCH 3/8] Fix db cleanup test The test wasn't running with CELERY_ALWAYS_EAGER so the clean task wasn't re-running itself to cleanup the other result --- cabot/cabotapp/tests/tests_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cabot/cabotapp/tests/tests_basic.py b/cabot/cabotapp/tests/tests_basic.py index ba34c50..f8146a7 100644 --- a/cabot/cabotapp/tests/tests_basic.py +++ b/cabot/cabotapp/tests/tests_basic.py @@ -988,7 +988,7 @@ class TestCleanUpTask(LocalTestCase): self.assertEqual(StatusCheckResult.objects.all().count(), initial_results + 2) tasks.clean_db(batch_size=1) - self.assertEqual(StatusCheckResult.objects.all().count(), initial_results + 1) + self.assertEqual(StatusCheckResult.objects.all().count(), initial_results) class TestMinimizeTargets(LocalTestCase): From fd5f0d6b48c3fb29d2345bf645ae08475deb4136 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 9 Feb 2017 21:37:26 +0000 Subject: [PATCH 4/8] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 057d5a0..07897d8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ master ------ +* Build docker image from alpine +* Refactor docker-compose files * Fix db_clean task failing on large results tables * Wait for docker containers to start in docker-entrypoint.sh * Update CABOT_PLUGINS_ENABLED to compatible plugin versions From 06716e983742e443715730ecc7094452b64ef68e Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Mon, 13 Feb 2017 14:17:07 +0000 Subject: [PATCH 5/8] Update travis.yml to use docker-compose-test.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 35677d0..43b02db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_script: script: - tox - - docker-compose run --rm -e SKIP_INIT=1 -e DATABASE_URL='sqlite://:memory:' web bash bin/test_with_coverage -v2 + - docker-compose -f docker-compose-test.yml run --rm test bash bin/test_with_coverage -v2 - docker-compose run --rm -e CABOT_SUPERUSER_USERNAME='admin' -e CABOT_SUPERUSER_PASSWORD='pass' web true after_success: From da2cbe85c02afea13e63910786aa729216c65eb1 Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Mon, 13 Feb 2017 14:24:49 +0000 Subject: [PATCH 6/8] Remove docker-compose-production.yml I think this should be moved to a separate repository --- docker-compose-production.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 docker-compose-production.yml diff --git a/docker-compose-production.yml b/docker-compose-production.yml deleted file mode 100644 index 83b86dc..0000000 --- a/docker-compose-production.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: '2' -services: - base: - extends: - file: docker-compose-base.yml - service: base - env_file: - - conf/production.env - - web: - extends: base - command: gunicorn cabot.wsgi:application --config gunicorn.conf - links: - - redis - - db - - worker: - extends: base - command: python manage.py celery worker -A cabot --loglevel=DEBUG --concurrency=16 -Ofair - links: - - redis - - db - - beat: - extends: base - command: python manage.py celery beat -A cabot --loglevel=DEBUG - links: - - redis - - db - - redis: - image: redis - - db: - image: postgres From f59d02926aa73ce7b6e088803a6582073655ac8e Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 16 Feb 2017 11:58:11 +0000 Subject: [PATCH 7/8] Move sqlalchemy to requirements-dev.txt It's only used for tests (to use sqlite as celery queue) --- requirements-dev.txt | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 576621c..bb69d77 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ coverage==4.2 django_coverage_plugin==1.3.1 mock==1.0.1 +sqlalchemy==1.1.5 ipdb diff --git a/requirements.txt b/requirements.txt index bcca038..750b7e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,4 +30,3 @@ twilio==3.4.1 wsgiref==0.1.2 python-dateutil==2.1 django-auth-ldap==1.2.6 -sqlalchemy==1.1.5 From 86b41dcfd16eb640671353fecbb7f0c2d3ddbaeb Mon Sep 17 00:00:00 2001 From: Frank Hamand Date: Thu, 16 Feb 2017 12:00:11 +0000 Subject: [PATCH 8/8] Add comment for wait_for_migrations hack --- docker-entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 1b1b9cd..a42a927 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -22,6 +22,8 @@ function wait_for_database {( function wait_for_migrations {( set +e for try in {1..60} ; do + # Kind of ugly but not sure if there's another way to determine if migrations haven't run + # migrate --list returns a checkbox list of migrations, empty checkboxes mean they haven't been run python manage.py migrate --list | grep "\[ \]" &> /dev/null || break echo "Waiting for database migrations to be run..." sleep 1