diff --git a/README.md b/README.md new file mode 100644 index 0000000..028f521 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Description + +This role configures an [Icinga2](https://icinga.com/products/icinga-2/) which is a monitoring tool which is used to manage canaries for Status websites as well as `statusd` nodes. + +# Usage + +The UI can be accessed via GitHub OAuth at: +https://canary.infra.status.im/ + +For more details read: +https://icinga.com/docs/icinga2/latest/ + +# Details + +The `statusd` canary comes from the `status-go` repository: +https://github.com/status-im/status-go/tree/develop/cmd/node-canary + +# Known Issues + +* UI can fail to auto-login when permissions on some files are wrong + - Usually it's wrong permissions on `/docker/icinga/etc/icingaweb2/authentication.ini` + - Or you might have to adjust ownership for whole `/docker/icinga/etc/icingaweb2` diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..f19d4a0 --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,65 @@ +--- +cont_state: started +cont_recreate: false +cont_restart: false + +icinga_cont_version: '2.11.4' +icinga_cont_image_mod_tag: '{{ icinga_cont_version }}' +icinga_cont_image_mod_name: 'statusteam/icinga2' +icinga_cont_name: 'icinga' +icinga_cont_vol: '/docker/{{ icinga_cont_name }}' + +icinga_api_cont_uid: 104 +icinga_api_cont_gid: 106 +icinga_web_cont_uid: 33 +# adjusted for dockremap +icinga_api_host_uid: '{{ 100000 + icinga_api_cont_uid }}' +icinga_api_host_gid: '{{ 100000 + icinga_api_cont_gid }}' +icinga_web_host_uid: '{{ 100000 + icinga_web_cont_uid }}' +icinga_cont_web_port: 80 +icinga_web_port: 8180 +icinga_api_port: 5665 +icinga_auto_login: admin +icinga_domain: 'canary.infra.status.im' +icinga_public_url: 'https://{{ icinga_domain }}/icingaweb2' +icinga_from_mail: 'icinga@mgun.status.im' +icinga_conf_d_path: '{{ icinga_cont_vol }}/etc/icinga2/conf.d' +icinga_consul_pass: 'TheSingleton' +# for discovering statusd-mail hosts +consul_url: 'http://localhost:8500/v1/catalog' + +db_cont_image: 'mysql/mysql-server:5.7' +db_cont_name: 'icinga-db' +db_cont_port: 3306 +db_cont_vol: '/docker/{{ db_cont_name }}' +db_cont_uid: 27 +db_host_uid: '{{ 100000 + db_cont_uid | int }}' + +# email sending settings +smtp_host: smtp.mailgun.org +smtp_port: 587 +smtp_from: 'alerts@mgun.status.im' +smtp_user: '{{ lookup("passwordstore", "cloud/MailGun/smtp-user") }}' +smtp_pass: '{{ lookup("passwordstore", "cloud/MailGun/smtp-pass") }}' + +# VictorOps notification settings +victorops_org_id: '{{ lookup("passwordstore", "cloud/VictorOps/org-id") }}' +victorops_service_key: '{{ lookup("passwordstore", "cloud/VictorOps/icinga-canary-key") }}' +victorops_contact_email: jakub@status.im +victorops_ack_back: true +victorops_deb_version: '1.4.20' +victorops_deb_file: 'victorops-nagios_{{ victorops_deb_version }}_all.deb' +victorops_deb_url: 'https://github.com/victorops/monitoring_tool_releases/releases/download/victorops-nagios-{{ victorops_deb_version }}/{{ victorops_deb_file }}' + +# go necessary to build status-go +golang_arch: 'go1.13.4.linux-amd64.tar.gz' +golang_md5: '823f651b12a578e54ee905292294bd69' + +# mailserver-canary settings +status_go_repo: 'https://github.com/status-im/status-go' +status_go_path: '/opt/status-im/status-go' +status_go_version: '3996aa7853f8274adb352e5629e123589a1a0671' + +statusd_bots_repo: 'https://github.com/status-im/statusd-bots.git' +statusd_bots_path: '/opt/status-im/statusd-bots' +statusd_bots_version: '0da56ebe56247ed106fd8ed4c9101b53f55f5482' diff --git a/files/general/constants.conf b/files/general/constants.conf new file mode 100644 index 0000000..7295d0e --- /dev/null +++ b/files/general/constants.conf @@ -0,0 +1,3 @@ +/* This file defines global extra constants */ + +const CustomPluginDir = "/var/tmp/nagios/plugins" diff --git a/files/general/groups.conf b/files/general/groups.conf new file mode 100644 index 0000000..9622b7b --- /dev/null +++ b/files/general/groups.conf @@ -0,0 +1,11 @@ +//object HostGroup "http" { +// display_name = "HTTP Checks" +// +// assign where match("http*", host.check_command) +//} +// +//object HostGroup "https" { +// display_name = "HTTPS Checks" +// +// assign where match("1", host.vars.http_ssl) +//} diff --git a/files/general/mailnode_canary.conf b/files/general/mailnode_canary.conf new file mode 100644 index 0000000..e62524c --- /dev/null +++ b/files/general/mailnode_canary.conf @@ -0,0 +1,50 @@ +object CheckCommand "mailserver_canary" { + /* we use the wrap to fix how icingaweb2 handles pipes in output */ + command = [ CustomPluginDir + "/node-canary-wrap" ] + + arguments = { + "-mailserver" = { + value = "$mailsrv_enode$" + description = "REQUIRED ARGUMENT: Mailserver enode address." + required = true + } + "-home-dir" = { + value = "$mailsrv_home_dir$" + description = "Home directory where state is stored." + } + "-log" = { + value = "$mailsrv_log_lvl$" + description = "ERROR, WARN, INFO, DEBUG, TRACE" + } + "-log-without-color" = { + set_if = "$mailsrv_log_no_color$" + description = "Disables log colors" + } + /* Optional arguments */ + //"-channel" = { + // value = "$mailsrv_channel$" + // description = "The public channel name to retrieve historic messages from." + //} + //"-period" = { + // value = "$mailsrv_period$" + // description = "How far in the past to request messages from mailserver, seconds." + //} + //"-shh.pow" = { + // value = "$mailsrv_shh_pow$" + // description = "PoW for messages to be added to queue, float." + //} + //"-shh.ttl" = { + // value = "$mailsrv_shh_ttl$" + // description = "Time to live for messages, in seconds." + //} + } + + vars.mailsrv_enode = null + vars.mailsrv_home_dir = "/tmp/mailsrv_canary_$host.name$" + vars.mailsrv_log_lvl = "ERROR" + vars.mailsrv_log_no_color = true + //vars.mailsrv_channel = "status" + //vars.mailsrv_period = "86400" + //vars.mailsrv_shh_pow = "0.001" + //vars.mailsrv_shh_ttl = "120" +} diff --git a/files/general/notifications.conf b/files/general/notifications.conf new file mode 100644 index 0000000..63aad88 --- /dev/null +++ b/files/general/notifications.conf @@ -0,0 +1,19 @@ +apply Notification "mail-icingaadmin" to Host { + import "mail-host-notification" + user_groups = host.vars.notification.mail.groups + users = host.vars.notification.mail.users + + vars.notification_logtosyslog = true + + assign where host.vars.notification.mail +} + +apply Notification "mail-icingaadmin" to Service { + import "mail-service-notification" + user_groups = host.vars.notification.mail.groups + users = host.vars.notification.mail.users + + vars.notification_logtosyslog = true + + assign where host.vars.notification.mail +} diff --git a/files/general/services.conf b/files/general/services.conf new file mode 100644 index 0000000..223aa57 --- /dev/null +++ b/files/general/services.conf @@ -0,0 +1,9 @@ +/* +apply Service "ping4" { + import "generic-service" + + check_command = "ping4" + + assign where host.address +} +*/ diff --git a/files/general/templates.conf b/files/general/templates.conf new file mode 100644 index 0000000..34b4d14 --- /dev/null +++ b/files/general/templates.conf @@ -0,0 +1,35 @@ +/** + * Provides default settings for hosts. By convention + * all hosts should import this template. + * + * The CheckCommand object `hostalive` is provided by + * the plugin check command templates. + * the plugin check command templates. + * Check the documentation for details. + */ +template Host "generic-host" { + max_check_attempts = 3 + check_interval = 1m + retry_interval = 30s + + check_command = "hostalive" +} + +/** + * Provides default settings for services. By convention + * all services should import this template. + */ +template Service "generic-service" { + max_check_attempts = 5 + check_interval = 1m + retry_interval = 30s +} + +/** + * Provides default settings for users. By convention + * all users should inherit from this template. + */ + +template User "generic-user" { + +} diff --git a/files/general/users.conf b/files/general/users.conf new file mode 100644 index 0000000..b7e0356 --- /dev/null +++ b/files/general/users.conf @@ -0,0 +1,17 @@ +object UserGroup "icingaadmins" { + display_name = "Icinga 2 Admin Group" +} + +object UserGroup "mailserveradmins" { + display_name = "Mailserver Admin Group" +} + +object User "icingaadmin" { + import "generic-user" + + display_name = "Icinga 2 Admin" + groups = [ "icingaadmins", "mailserveradmins" ] + + enable_notifications = true + email = "jakub@status.im" +} diff --git a/files/general/whispernode_canary.conf b/files/general/whispernode_canary.conf new file mode 100644 index 0000000..c98b816 --- /dev/null +++ b/files/general/whispernode_canary.conf @@ -0,0 +1,50 @@ +object CheckCommand "whisper_canary" { + /* we use the wrap to fix how icingaweb2 handles pipes in output */ + command = [ CustomPluginDir + "/node-canary-wrap" ] + + arguments = { + "-staticnode" = { + value = "$whisper_enode$" + description = "REQUIRED ARGUMENT: Whisper enode address." + required = true + } + "-home-dir" = { + value = "$whisper_home_dir$" + description = "Home directory where state is stored." + } + "-log" = { + value = "$whisper_log_lvl$" + description = "ERROR, WARN, INFO, DEBUG, TRACE" + } + "-log-without-color" = { + set_if = "$whisper_log_no_color$" + description = "Disables log colors" + } + /* Optional arguments */ + //"-channel" = { + // value = "$whisper_channel$" + // description = "The public channel name to retrieve historic messages from." + //} + //"-period" = { + // value = "$whisper_period$" + // description = "How far in the past to request messages from mailserver, seconds." + //} + //"-shh.pow" = { + // value = "$whisper_shh_pow$" + // description = "PoW for messages to be added to queue, float." + //} + //"-shh.ttl" = { + // value = "$whisper_shh_ttl$" + // description = "Time to live for messages, in seconds." + //} + } + + vars.whisper_enode = null + vars.whisper_home_dir = "/tmp/whisper_canary_$service.name$_$host.name$" + vars.whisper_log_lvl = "ERROR" + vars.whisper_log_no_color = true + //vars.whisper_channel = "status" + //vars.whisper_period = "86400" + //vars.whisper_shh_pow = "0.001" + //vars.whisper_shh_ttl = "120" +} diff --git a/files/general/x_check_mail_canary.conf b/files/general/x_check_mail_canary.conf new file mode 100644 index 0000000..ac215be --- /dev/null +++ b/files/general/x_check_mail_canary.conf @@ -0,0 +1,39 @@ +object CheckCommand "xcheck_mailserver" { + /* we use the wrap to fix how icingaweb2 handles pipes in output */ + command = [ CustomPluginDir + "/x-check-mail-wrap" ] + + arguments = { + "--fleet" = { + value = "$xcheck_fleet$" + description = "Cluster fleet to connect to." + required = true + } + "--datadir" = { + value = "$xcheck_datadir$" + description = "Directory for sockets and temporary files." + required = true + } + "--privkey" = { + value = "$xcheck_privkey$" + description = "Hexadecimal of the private key to use for node." + required = true + } + "--channels" = { + value = "$xcheck_channels$" + description = "Names of Status channels to check." + required = true + } + "--duration" = { + value = "$xcheck_duration$" + description = "length of time span from now (default 24h0m0s)" + } + "--verbosity" = { + value = "$xcheck_verbosity$" + description = "Verbosity level: crit, error, warn, info, debug" + } + } + + vars.xcheck_datadir = "/tmp/x-check-mailserver" + vars.xcheck_duration = "24h0m0s" + vars.xcheck_verbosity = "warn" +} diff --git a/files/node-canary-wrap.sh b/files/node-canary-wrap.sh new file mode 100644 index 0000000..842188d --- /dev/null +++ b/files/node-canary-wrap.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -o pipefail +# This is necessary because Icingaweb2 hates pipes in output +SCRIPT_DIR=$(cd `dirname $0` && pwd) + +${SCRIPT_DIR}/node-canary $@ 2>&1 | sed 's/^[A-Z ]\+\[.*\] //' +EXIT_CODE=$? + +if [[ ${EXIT_CODE} -ne 0 ]]; then + echo "Failed with args: $@" + echo "Exit code: ${EXIT_CODE}" + exit 2 +fi diff --git a/files/pages/https_service.conf b/files/pages/https_service.conf new file mode 100644 index 0000000..b6c739f --- /dev/null +++ b/files/pages/https_service.conf @@ -0,0 +1,22 @@ +object ServiceGroup "website" {} + +apply Service for (http_vhost => config in host.vars.http_vhosts) { + import "generic-service" + + groups = [ "website" ] + + /* fix for applying this to canary host */ + assign where host.name == "Website Availability" + + check_command = "http" + + vars.http_ssl = "1" + vars.http_uri = "/" + vars.http_expect = "200" + vars.http_onredirect = "follow" + vars.http_vhost = http_vhost + vars.http_address = http_vhost + vars.http_sni = true + + vars += config +} diff --git a/files/pages/website_availability.conf b/files/pages/website_availability.conf new file mode 100644 index 0000000..69d57e5 --- /dev/null +++ b/files/pages/website_availability.conf @@ -0,0 +1,38 @@ +object Host "Website Availability" { + import "generic-host" + address = "127.0.0.1" + + vars.http_vhosts["status.im"] = { } + vars.http_vhosts["our.status.im"] = { } + vars.http_vhosts["discuss.status.im"] = { } + vars.http_vhosts["forum.vac.dev"] = { } + vars.http_vhosts["nimbus.team"] = { } + vars.http_vhosts["keycard.tech"] = { } + vars.http_vhosts["get.keycard.tech"] = { } + vars.http_vhosts["boards.status.im"] = { } + vars.http_vhosts["notes.status.im"] = { } + vars.http_vhosts["gh.status.im"] = { } + vars.http_vhosts["metrics.status.im"] = { } + vars.http_vhosts["ipfs.status.im"] = { } + vars.http_vhosts["nix-cache.status.im"] = { http_expect = "302" } + vars.http_vhosts["blog.status.im"] = { http_expect = "302" } + + # Internal services + vars.http_vhosts["grafana.infra.status.im"] = { http_expect = "302" } + vars.http_vhosts["alerts.infra.status.im"] = { http_expect = "403" } + vars.http_vhosts["canary.infra.status.im"] = { http_expect = "403" } + vars.http_vhosts["consul.infra.status.im"] = { http_expect = "403" } + vars.http_vhosts["logs-es.infra.status.im"] = { http_expect = "403" } + vars.http_vhosts["kibana.infra.status.im"] = { http_expect = "403" } + vars.http_vhosts["prometheus.infra.status.im"] = { http_expect = "403" } + + # GitHub webhooks + vars.http_vhosts["status-github-bot.status.im"] = { http_uri = "/ping" } + vars.http_vhosts["figma-diff-probot.status.im"] = { http_uri = "/ping" } + vars.http_vhosts["packages-check-bot.status.im"] = { http_uri = "/ping" } + vars.http_vhosts["deps-lock-snitch-bot.status.im"] = { http_uri = "/ping" } + + vars.notification["mail"] = { + groups = [ "icingaadmins" ] + } +} diff --git a/files/x-check-mail-wrap.sh b/files/x-check-mail-wrap.sh new file mode 100644 index 0000000..410cf1b --- /dev/null +++ b/files/x-check-mail-wrap.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -o pipefail +# This is necessary because Icingaweb2 hates pipes in output +SCRIPT_DIR=$(cd `dirname $0` && pwd) + +${SCRIPT_DIR}/x-check-mailserver $@ 2>&1 | sed 's/^[A-Z ]\+\[.*\] //' +EXIT_CODE=$? + +if [[ $EXIT_CODE -ne 0 ]]; then + echo "Failed with args: $@" + echo "Exit code: $EXIT_CODE" + exit 1 +fi diff --git a/files/xcheck/mailserver_service.conf b/files/xcheck/mailserver_service.conf new file mode 100644 index 0000000..e052285 --- /dev/null +++ b/files/xcheck/mailserver_service.conf @@ -0,0 +1,21 @@ +object ServiceGroup "x-check" {} + +apply Service for (xcheck_mail => config in host.vars.xcheck_mail) { + import "generic-service" + + check_interval = 10m + + groups = [ "x-check" ] + + /* fix for applying this to canary host */ + assign where host.name == "X-Check Mailserver" + + check_command = "xcheck_mailserver" + + vars.xcheck_fleet = "eth.prod" + vars.xcheck_channels = "status" + /* This key is static to avoid generating more "fake" peers */ + vars.xcheck_privkey = "1111111111111111111111111111111111111111111111111111111111111111" + + vars += config +} diff --git a/files/xcheck/x_check_mailserver.conf b/files/xcheck/x_check_mailserver.conf new file mode 100644 index 0000000..30acba6 --- /dev/null +++ b/files/xcheck/x_check_mailserver.conf @@ -0,0 +1,21 @@ +object Host "X-Check Mailserver" { + import "generic-host" + address = "127.0.0.1" + + vars.xcheck_mail["eth.prod:status"] = { xcheck_fleet="eth.prod", + xcheck_channels="status" } + vars.xcheck_mail["eth.prod:support"] = { xcheck_fleet="eth.prod", + xcheck_channels="support" } + vars.xcheck_mail["eth.staging:status"] = { xcheck_fleet="eth.staging", + xcheck_channels="status" } + vars.xcheck_mail["eth.staging:support"] = { xcheck_fleet="eth.staging", + xcheck_channels="support" } + vars.xcheck_mail["eth.test:status"] = { xcheck_fleet="eth.test", + xcheck_channels="status" } + vars.xcheck_mail["eth.test:support"] = { xcheck_fleet="eth.test", + xcheck_channels="support" } + + vars.notification["mail"] = { + groups = [ "mailserveradmins" ] + } +} diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..bd6da46 --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Save iptables rules + shell: iptables-save > /etc/iptables/rules.v4 diff --git a/tasks/consul.yml b/tasks/consul.yml new file mode 100644 index 0000000..7c9227c --- /dev/null +++ b/tasks/consul.yml @@ -0,0 +1,19 @@ +--- +- name: Create Consul service definition + include_role: name=consul-service + vars: + consul_config_name: '{{ icinga_cont_name }}-api' + consul_services: + - name: '{{ icinga_cont_name }}-api' + tags: ['canary'] + # we advertise the port with basic auth + port: '{{ icinga_api_port }}' + address: '{{ ansible_local.tinc.vpn_ip }}' + checks: + - id: '{{ icinga_cont_name }}-api-health' + name: ElasticHQ Healthcheck + type: http + http: 'https://localhost:{{ icinga_api_port }}/v1/status/IcingaApplication' + tls_skip_verify: true + header: + Authorization: ['Basic {{ ("consul:" + icinga_consul_pass) | b64encode }}'] diff --git a/tasks/container.yml b/tasks/container.yml new file mode 100644 index 0000000..5106276 --- /dev/null +++ b/tasks/container.yml @@ -0,0 +1,63 @@ +--- +- name: Start MySQL container + docker_container: + name: '{{ db_cont_name }}' + image: '{{ db_cont_image }}' + pull: true + restart_policy: always + state: '{{ cont_state }}' + recreate: '{{ cont_recreate }}' + restart: '{{ cont_restart }}' + env: + MYSQL_ROOT_PASSWORD: '{{ lookup("passwordstore", "services/icinga/db-root-pass")}}' + MYSQL_DATABASE: 'icinga' + MYSQL_USER: '{{ lookup("passwordstore", "services/icinga/db-user")}}' + MYSQL_PASSWORD: '{{ lookup("passwordstore", "services/icinga/db-pass")}}' + MYSQL_ROOT_HOST: '%' + ports: + - '0.0.0.0:{{ db_cont_port }}:{{ db_cont_port }}' + volumes: + - '{{ db_cont_vol }}/data:/var/lib/mysql' + +- name: Start Icinga2 container + docker_container: + name: '{{ icinga_cont_name }}' + image: '{{ icinga_cont_image_mod_name }}:{{ icinga_cont_image_mod_tag }}' + pull: false + restart_policy: always + state: '{{ cont_state }}' + recreate: '{{ cont_recreate }}' + restart: '{{ cont_restart }}' + hostname: '{{ hostname }}' + links: + - '{{ db_cont_name }}:db' + env: + ICINGAWEB2_ADMIN_USER: '{{ lookup("passwordstore", "services/icinga/user") }}' + ICINGAWEB2_ADMIN_PASS: '{{ lookup("passwordstore", "services/icinga/pass") }}' + DEFAULT_MYSQL_HOST: 'db' + DEFAULT_MYSQL_PORT: '{{ db_cont_port | string }}' + DEFAULT_MYSQL_USER: '{{ lookup("passwordstore", "services/icinga/db-user") }}' + DEFAULT_MYSQL_PASS: '{{ lookup("passwordstore", "services/icinga/db-pass") }}' + MYSQL_ROOT_PASSWORD: '{{ lookup("passwordstore", "services/icinga/db-root-pass") }}' + ports: + - '0.0.0.0:{{ icinga_web_port }}:{{ icinga_cont_web_port }}' + - '0.0.0.0:{{ icinga_api_port }}:{{ icinga_api_port }}' + mounts: + - target: /tmp + type: 'tmpfs' + tmpfs_size: '131072K' + volumes: + - '{{ icinga_cont_vol }}/etc/icinga2:/etc/icinga2' + - '{{ icinga_cont_vol }}/etc/icingaweb2:/etc/icingaweb2' + - '{{ icinga_cont_vol }}/plugins:/var/tmp/nagios/plugins' + - '{{ icinga_cont_vol }}/data:/var/lib/icinga2' + - '{{ icinga_cont_vol }}/logs:/var/log/icinga2' + - '{{ icinga_cont_vol }}/conf/ssmtp.conf:/etc/ssmtp/ssmtp.conf' + - '{{ icinga_cont_vol }}/conf/apache_icinga.conf:/etc/apache2/conf-enabled/icingaweb2.conf' + +- name: Wait for the Icinga API port to become available + wait_for: + host: '127.0.0.1' + port: '{{ icinga_api_port }}' + delay: 5 + state: drained diff --git a/tasks/dependencies.yml b/tasks/dependencies.yml new file mode 100644 index 0000000..68ec3ba --- /dev/null +++ b/tasks/dependencies.yml @@ -0,0 +1,43 @@ +--- +- name: Uninstall Go from APT + apt: + name: golang + state: absent + autoremove: true + tags: ['icinga::dependencies'] + +- name: Remove old Go installation + file: + path: '/usr/local/go' + state: absent + tags: ['icinga::dependencies'] + +- name: Download Go tarball + get_url: + url: 'https://dl.google.com/go/{{ golang_arch }}' + dest: '/tmp/{{ golang_arch }}' + checksum: 'md5:{{ golang_md5 }}' + tags: ['icinga::dependencies'] + +- name: Extract Go tarball + unarchive: + src: '/tmp/{{ golang_arch }}' + dest: '/usr/local' + remote_src: yes + environment: + PATH: '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' + tags: ['icinga::dependencies'] + +- name: Symlink Go binary + file: + src: '/usr/local/go/bin/go' + path: '/usr/local/bin/go' + state: link + force: true + tags: ['icinga::dependencies'] + +- name: Remove Go tarball + file: + path: '/tmp/{{ golang_arch }}' + state: absent + tags: ['icinga::dependencies'] diff --git a/tasks/firewall.yml b/tasks/firewall.yml new file mode 100644 index 0000000..f24cf30 --- /dev/null +++ b/tasks/firewall.yml @@ -0,0 +1,14 @@ +--- +- name: Enable Icinga2 Web UI + iptables: + comment: 'Icinga2 Web UI' + chain: INPUT + jump: ACCEPT + source: '0.0.0.0/0' + protocol: 'tcp' + destination_port: '{{ item }}' + with_items: + - 80 + - 443 + notify: + - Save iptables rules diff --git a/tasks/hosts.yml b/tasks/hosts.yml new file mode 100644 index 0000000..fd75bd4 --- /dev/null +++ b/tasks/hosts.yml @@ -0,0 +1,52 @@ +--- +- name: Get data centers + uri: + url: '{{ consul_url }}/datacenters' + register: data_centers + +- name: Discover mailnode services + uri: + url: '{{ consul_url }}/service/statusd-mail?dc={{ item }}' + register: mail_servers + with_items: '{{ data_centers.json }}' + +- name: Discover whisper services + uri: + url: '{{ consul_url }}/service/statusd-whisper?dc={{ item }}' + register: whisper_servers + with_items: '{{ data_centers.json }}' + +- name: Discover eth fleet hosts + uri: + url: '{{ consul_url }}/nodes?node-meta=env:eth&dc={{ item }}' + register: statusd_hosts + with_items: '{{ data_centers.json }}' + +- name: Combine data centers into one list + set_fact: + statusd_hosts: '{{ statusd_hosts.results | sum(attribute="json", start=[]) }}' + mail_servers: '{{ mail_servers.results | sum(attribute="json", start=[]) }}' + whisper_servers: '{{ whisper_servers.results | sum(attribute="json", start=[]) }}' + +- name: Merge all statusd services + set_fact: + statusd_services: '{{ whisper_servers + mail_servers }}' + +- name: Extract tags from services list + set_fact: + host_tags: '{{ statusd_hosts | map(attribute="Meta") | map(attribute="stage") | list | unique }}' + service_tags: '{{ statusd_services | sum(attribute="ServiceTags", start=[]) | unique }}' + +- name: Create folder for hosts + file: + path: '{{ icinga_conf_d_path }}/hosts' + state: directory + owner: '{{ icinga_api_host_uid }}' + group: docker + +- name: Generate host definitions for eth fleet + template: + src: icinga_hosts.conf.j2 + dest: '{{ icinga_conf_d_path }}/hosts/eth_fleets.conf' + owner: '{{ icinga_api_host_uid }}' + group: docker diff --git a/tasks/image.yml b/tasks/image.yml new file mode 100644 index 0000000..474a531 --- /dev/null +++ b/tasks/image.yml @@ -0,0 +1,21 @@ +--- +# VictorOps integration requires extra packages so we update the image +- name: Create folder for building the docker image + file: + path: '/var/tmp/icinga' + state: directory + +- name: Template the Dockerfile for Icinga + template: + src: 'Dockerfile.j2' + dest: '/var/tmp/icinga/Dockerfile' + mode: 0755 + register: dockerfiles + +- name: Install required packages in Icinga2 docker image + docker_image: + name: '{{ icinga_cont_image_mod_name }}' + tag: '{{ icinga_cont_image_mod_tag }}' + force: '{{ dockerfiles.changed | default(false) }}' + build: + path: '/var/tmp/icinga' diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..7704ca4 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- import_tasks: dependencies.yml +- import_tasks: pre_config.yml +- import_tasks: image.yml +- import_tasks: container.yml +- import_tasks: plugins.yml +- import_tasks: node_canary.yml +- import_tasks: x_check_mailserver.yml +- import_tasks: hosts.yml +- import_tasks: post_config.yml +- import_tasks: firewall.yml +- import_tasks: consul.yml diff --git a/tasks/node_canary.yml b/tasks/node_canary.yml new file mode 100644 index 0000000..a79a079 --- /dev/null +++ b/tasks/node_canary.yml @@ -0,0 +1,43 @@ +--- +# Build node-canary from git repo ------------------------------- +- name: Download status-go to deploy node-canary + git: + repo: '{{ status_go_repo }}' + dest: '{{ status_go_path }}' + version: '{{ status_go_version }}' + depth: 1 + update: true + umask: '0000' + register: repo + tags: ['icinga::node-canary'] + +# We use Buster because of host newer GLibC version +- name: Build node-canary + docker_container: + name: 'build-node-canary' + image: 'golang:1.14-buster' + detach: false + command: 'make node-canary' + working_dir: '/repo' + volumes: + - '{{ status_go_path }}:/repo' + tags: ['icinga::node-canary'] + +- name: Copy over the node-canary binary + copy: + remote_src: true + src: '{{ status_go_path }}/build/bin/node-canary' + dest: '{{ icinga_cont_vol }}/plugins/node-canary' + owner: '{{ icinga_api_host_uid }}' + group: docker + mode: 0775 + tags: ['icinga::node-canary'] + +- name: Create a wrapper for cleaning pipes from canary output + copy: + src: 'node-canary-wrap.sh' + dest: '{{ icinga_cont_vol }}/plugins/node-canary-wrap' + owner: '{{ icinga_api_host_uid }}' + group: docker + mode: 0775 + tags: ['icinga::node-canary'] diff --git a/tasks/plugins.yml b/tasks/plugins.yml new file mode 100644 index 0000000..b63542b --- /dev/null +++ b/tasks/plugins.yml @@ -0,0 +1,22 @@ +--- +# This tempalate comes from /opt/victorops/nagios_plugin/icinga2_conf/victorops.conf +# For more details see: https://help.victorops.com/knowledge-base/victorops-icinga-integration/ +- name: Generate the VictorOps configuration + template: + src: victorops.conf.j2 + dest: '{{ icinga_conf_d_path }}/victorops.status-im.conf' + +- name: Fix permissions on the /etc/icinga*2 folders + file: + path: '{{ item }}' + owner: '{{ icinga_api_host_uid }}' + group: '{{ icinga_api_host_gid }}' + recurse: true + with_items: + - '{{ icinga_cont_vol }}/etc/icinga2' + - '{{ icinga_cont_vol }}/etc/icingaweb2' + +- name: Fix permissions on the conf.d folder + file: + path: '{{ icinga_conf_d_path }}' + mode: 0755 diff --git a/tasks/post_config.yml b/tasks/post_config.yml new file mode 100644 index 0000000..a90bff9 --- /dev/null +++ b/tasks/post_config.yml @@ -0,0 +1,81 @@ +--- +# We have to update it after container was started because +# the container generates files in /etc/icingaweb2 +# and if one of them is there it fails +# +# WARNING: Wrong permissions here will cause login issues +# Details: https://github.com/Icinga/icingaweb2/issues/3737 +- name: Enable autologin in Icinga Web UI + template: + src: icingaweb_auth.ini.j2 + dest: '{{ icinga_cont_vol }}/etc/icingaweb2/authentication.ini' + owner: '{{ icinga_api_host_uid }}' + group: '{{ icinga_api_host_gid }}' + mode: 755 + group: docker + register: auth + +- name: Configure general settings + copy: + src: '{{ item }}' + dest: '{{ icinga_cont_vol }}/etc/icinga2/conf.d/' + owner: '{{ icinga_api_host_uid }}' + group: docker + register: general + with_fileglob: + - files/general/* + +- name: Generate general settings from templates + template: + src: '{{ item }}' + dest: '{{ icinga_conf_d_path }}/{{ item | basename | replace(".j2", "") }}' + with_fileglob: + - templates/icinga/* + +- name: Fix permissions on the /etc/icinga*2 folders + file: + path: '{{ item }}' + owner: '{{ icinga_api_host_uid }}' + group: '{{ icinga_api_host_gid }}' + recurse: true + with_items: + - '{{ icinga_cont_vol }}/etc/icinga2' + - '{{ icinga_cont_vol }}/etc/icingaweb2' + +- name: Create folder for pages + file: + path: '{{ icinga_cont_vol }}/etc/icinga2/conf.d/pages' + owner: '{{ icinga_api_host_uid }}' + group: docker + state: directory + +- name: Configure pages checks + copy: + src: '{{ item }}' + dest: '{{ icinga_cont_vol }}/etc/icinga2/conf.d/pages/' + owner: '{{ icinga_api_host_uid }}' + group: docker + register: pages + with_fileglob: + - files/pages/* + +- name: Create folder for xcheck + file: + path: '{{ icinga_cont_vol }}/etc/icinga2/conf.d/xcheck' + owner: '{{ icinga_api_host_uid }}' + group: docker + state: directory + +- name: Configure xcheck checks + copy: + src: '{{ item }}' + dest: '{{ icinga_cont_vol }}/etc/icinga2/conf.d/xcheck/' + owner: '{{ icinga_api_host_uid }}' + group: docker + register: xcheck + with_fileglob: + - files/xcheck/* + +- name: Reload the icinga container + command: docker exec {{ icinga_cont_name }} supervisorctl reload icinga2 + when: auth.changed or general.changed or pages.changed diff --git a/tasks/pre_config.yml b/tasks/pre_config.yml new file mode 100644 index 0000000..c8cf97d --- /dev/null +++ b/tasks/pre_config.yml @@ -0,0 +1,44 @@ +--- +- name: Create directories for Icinga + file: + path: '{{ item }}' + state: directory + owner: '{{ icinga_api_host_uid }}' + group: '{{ icinga_api_host_gid }}' + recurse: true + with_items: + - '{{ icinga_cont_vol }}/etc' + - '{{ icinga_cont_vol }}/conf' + - '{{ icinga_cont_vol }}/logs' + - '{{ icinga_cont_vol }}/data' + - '{{ icinga_cont_vol }}/plugins' + +- name: Fix permissions for webicinga2 + file: + path: '{{ icinga_cont_vol }}/etc/icingaweb2' + state: directory + owner: '{{ icinga_api_host_uid }}' + group: '{{ icinga_api_host_gid }}' + recurse: true + +- name: Create directories for MySQL + file: + path: '{{ db_cont_vol }}/data' + state: directory + owner: '{{ db_host_uid }}' + group: docker + recurse: true + +- name: Generate the SMTP configuration + template: + src: ssmtp.conf.j2 + dest: '{{ icinga_cont_vol }}/conf/ssmtp.conf' + owner: '{{ icinga_api_host_uid }}' + group: docker + +- name: Generate the Apache configuration for autologin + template: + src: apache_icinga.conf.j2 + dest: '{{ icinga_cont_vol }}/conf/apache_icinga.conf' + owner: '{{ icinga_web_host_uid }}' + group: '{{ icinga_api_host_gid }}' diff --git a/tasks/x_check_mailserver.yml b/tasks/x_check_mailserver.yml new file mode 100644 index 0000000..b3d0121 --- /dev/null +++ b/tasks/x_check_mailserver.yml @@ -0,0 +1,44 @@ +--- +# Build x-check-mailserver from git repo ----------------------- +- name: Download statusd-bots to deploy x-check-mailserver + git: + repo: '{{ statusd_bots_repo }}' + dest: '{{ statusd_bots_path }}' + version: '{{ statusd_bots_version }}' + depth: 1 + update: true + force: true + umask: '0000' + register: repo + tags: ['icinga::x-check'] + +# We use Buster because of host newer GLibC version +- name: Build x-check-mailserver + docker_container: + name: 'build-x-check-mailserver' + image: 'golang:1.14-buster' + detach: false + command: 'make x-check-mailserver' + working_dir: '/repo' + volumes: + - '{{ statusd_bots_path }}:/repo' + tags: ['icinga::x-check'] + +- name: Copy over the node-canary binary + copy: + remote_src: true + src: '{{ statusd_bots_path }}/bin/x-check-mailserver' + dest: '{{ icinga_cont_vol }}/plugins/x-check-mailserver' + owner: '{{ icinga_api_host_uid }}' + group: docker + mode: 0775 + tags: ['icinga::x-check'] + +- name: Create a wrapper for cleaning pipes from canary output + copy: + src: 'x-check-mail-wrap.sh' + dest: '{{ icinga_cont_vol }}/plugins/x-check-mail-wrap' + owner: '{{ icinga_api_host_uid }}' + group: docker + mode: 0775 + tags: ['icinga::x-check'] diff --git a/templates/Dockerfile.j2 b/templates/Dockerfile.j2 new file mode 100644 index 0000000..041ac20 --- /dev/null +++ b/templates/Dockerfile.j2 @@ -0,0 +1,16 @@ +FROM jordan/icinga2:{{ icinga_cont_version }} + +MAINTAINER Jakub SokoĊ‚owski + +# Install VictorOps plugin for sending alerts +# https://help.victorops.com/knowledge-base/victorops-icinga-integration/ +RUN wget {{ victorops_deb_url }} \ + && dpkg -i {{ victorops_deb_file }} \ + && rm {{ victorops_deb_file }} + +# we run our own mysql in a separate container +RUN rm /etc/supervisor/conf.d/mysql.conf + +LABEL source="https://github.com/status-im/infra-hq" +LABEL description="Modified Icinga2 Image for use with VictorOps" +LABEL maintainer="jakub@status.im" diff --git a/templates/apache_icinga.conf.j2 b/templates/apache_icinga.conf.j2 new file mode 100644 index 0000000..557ee95 --- /dev/null +++ b/templates/apache_icinga.conf.j2 @@ -0,0 +1,26 @@ +Alias /icingaweb2 "/usr/share/icingaweb2/public" + + + Options SymLinksIfOwnerMatch + AllowOverride None + + SetEnv ICINGAWEB_CONFIGDIR "/etc/icingaweb2" + SetEnv REMOTE_USER {{ icinga_auto_login }} + + EnableSendfile Off + + + RewriteEngine on + RewriteBase /icingaweb2/ + RewriteCond %{REQUEST_FILENAME} -s [OR] + RewriteCond %{REQUEST_FILENAME} -l [OR] + RewriteCond %{REQUEST_FILENAME} -d + RewriteRule ^.*$ - [NC,L] + RewriteRule ^.*$ index.php [NC,L] + + + + DirectoryIndex error_norewrite.html + ErrorDocument 404 /error_norewrite.html + + diff --git a/templates/api-users.conf.j2 b/templates/api-users.conf.j2 new file mode 100644 index 0000000..8331703 --- /dev/null +++ b/templates/api-users.conf.j2 @@ -0,0 +1,12 @@ +/** + * The APIUser objects are used for authentication against the API. + */ +object ApiUser "root" { + password = "8b68ef3845222d43" + permissions = [ "*" ] +} + +object ApiUser "consul" { + password = "{{ icinga_consul_pass }}" + permissions = [ "v1/status/IcingaApplication" ] +} diff --git a/templates/icinga/api-users.conf.j2 b/templates/icinga/api-users.conf.j2 new file mode 100644 index 0000000..5f61e31 --- /dev/null +++ b/templates/icinga/api-users.conf.j2 @@ -0,0 +1,12 @@ +/** + * The APIUser objects are used for authentication against the API. + */ +object ApiUser "root" { + password = "8b68ef3845222d43" + permissions = [ "*" ] +} + +object ApiUser "consul" { + password = "{{ icinga_consul_pass }}" + permissions = [ "status/*" ] +} diff --git a/templates/icinga/mail.conf.j2 b/templates/icinga/mail.conf.j2 new file mode 100644 index 0000000..6e6943a --- /dev/null +++ b/templates/icinga/mail.conf.j2 @@ -0,0 +1,43 @@ +/** + * Provides default settings for host notifications. + * By convention all host notifications should import + * this template. + */ +template Notification "mail-host-notification" { + command = "mail-host-notification" + + states = [ Up, Down ] + types = [ Problem, Acknowledgement, Recovery, Custom, + FlappingStart, FlappingEnd, + DowntimeStart, DowntimeEnd, DowntimeRemoved ] + + vars += { + notification_icingaweb2url = "{{ icinga_public_url }}" + notification_from = "Icinga Host Monitoring <{{ icinga_from_mail }}>" + notification_logtosyslog = false + } + + period = "24x7" +} + +/** + * Provides default settings for service notifications. + * By convention all service notifications should import + * this template. + */ +template Notification "mail-service-notification" { + command = "mail-service-notification" + + states = [ OK, Warning, Critical, Unknown ] + types = [ Problem, Acknowledgement, Recovery, Custom, + FlappingStart, FlappingEnd, + DowntimeStart, DowntimeEnd, DowntimeRemoved ] + + vars += { + notification_icingaweb2url = "{{ icinga_public_url }}" + notification_from = "Icinga Service Monitoring <{{ icinga_from_mail }}>" + notification_logtosyslog = false + } + + period = "24x7" +} diff --git a/templates/icinga_host_tags.conf.j2 b/templates/icinga_host_tags.conf.j2 new file mode 100644 index 0000000..e69de29 diff --git a/templates/icinga_hosts.conf.j2 b/templates/icinga_hosts.conf.j2 new file mode 100644 index 0000000..4d70cb1 --- /dev/null +++ b/templates/icinga_hosts.conf.j2 @@ -0,0 +1,40 @@ +{% for tag in host_tags %} +object HostGroup "eth.{{ tag }}" {} +{% endfor %} + +{% for tag in service_tags %} +object ServiceGroup "{{ tag }}" {} +{% endfor %} + +{% for host in statusd_hosts %} +object Host "{{ host.Node }}" { + import "generic-host" + address = "{{ host.Address }}" + groups = [ "eth.{{ host.Meta.stage }}" ] +} + +{% endfor %} + +{% for srvc in mail_servers %} +object Service "{{ srvc.ServiceID }}" { + import "generic-service" + check_command = "mailserver_canary" + + host_name = "{{ srvc.Node }}" + groups = {{ srvc.ServiceTags | to_json }} + vars.mailsrv_enode = "{{ srvc.ServiceMeta.node_enode }}" +} + +{% endfor %} + +{% for srvc in whisper_servers %} +object Service "{{ srvc.ServiceID }}" { + import "generic-service" + check_command = "whisper_canary" + + host_name = "{{ srvc.Node }}" + groups = {{ srvc.ServiceTags | to_json }} + vars.whisper_enode = "{{ srvc.ServiceMeta.node_enode }}" +} + +{% endfor %} diff --git a/templates/icingaweb_auth.ini.j2 b/templates/icingaweb_auth.ini.j2 new file mode 100644 index 0000000..acc4b32 --- /dev/null +++ b/templates/icingaweb_auth.ini.j2 @@ -0,0 +1,6 @@ +[icingaweb2] +backend = "db" +resource = "icingaweb_db" + +[autologin] +backend = external diff --git a/templates/ssmtp.conf.j2 b/templates/ssmtp.conf.j2 new file mode 100644 index 0000000..72de53c --- /dev/null +++ b/templates/ssmtp.conf.j2 @@ -0,0 +1,7 @@ +root={{ smtp_from }} +mailhub={{ smtp_host }}:{{ smtp_port }} +# WARNING: No special chars! +AuthUser={{ smtp_user }} +AuthPass={{ smtp_pass }} +UseSTARTTLS=YES +FromLineOverride=NO diff --git a/templates/victorops.conf.j2 b/templates/victorops.conf.j2 new file mode 100644 index 0000000..f89b97f --- /dev/null +++ b/templates/victorops.conf.j2 @@ -0,0 +1,227 @@ +#------------------------------------------------------------------------------------------------ +########################## ACCOUNT CONFIGURATION ########################## +## These identify your alerts to VictorOps. The values for these fields are assigned to you +## by VictorOps and can be found in the Icinga section of the integrations page. +## +const VictorOps_ORGANIZATION_ID = "{{ victorops_org_id | mandatory }}" +const VictorOps_ORGANIZATION_KEY = "{{ victorops_service_key | mandatory }}" + +########################## FAILSAFE EMAIL CONFIGURATION ########################## +## The backup alert email address. If for any reason the plugin is unable to contact +## the VictorOps URL, an alert email will be sent to this address. You may configure multiple +## addresses by separating them with spaces and enclosing the whole thing in single quotes: +## "me@mydomain.com you@mydomain.com him@mydomain.com" +const VictorOps_CONTACTEMAIL = "{{ victorops_contact_email | mandatory }}" + +## The maximum amount of time (in seconds) that alerts will be allowed to remain in +## the queue before the alert is sent to the contact address above. +const VictorOps_MAX_SEND_DELAY = "60" + +########################## ROUTING CONFIGURATION ########################## +# This default user and notification apply configuration will send all host and service alerts +# to VictorOps via the VictorOps user. If you wish to use VictorOps routing keys, create a +# user for each routing key, and create appropriate notification apply rules to those users. +# See the VictorOps Icinga2 knowledge base for more information. + +apply Notification "notify-victorops-host" to Host { + import "notify-victorops-host" + assign where match("*", host.name) +} + +apply Notification "notify-victorops-service" to Service { + import "notify-victorops-service" + assign where match("*", service.name) + ignore where match("victorops-forwarder", service.name) +} + + +########################## ACK-BACK CONFIGURATION ########################## +const VictorOps_enable_cmd_poll = {{ victorops_ack_back | to_json }} +#------------------------------------------------------------------------------------------------ + +# Other required configuration... + +## Identifies this Icinga host to VictorOps. +const VictorOps_MONITOR_NAME = "{{ icinga_domain | default("Icinga Canaries") }}" + +# Probably in these locations: +const IcingaCmdPipe = "/var/run/icinga2/cmd/icinga2.cmd" +const IcingaStatusFile = "/var/cache/icinga2/status.dat" +const printf_executable = "/usr/bin/printf" +const mail_executable = "/usr/bin/mail" + +## These should not need modification +const VictorOps_PLUGIN_HOME = "/opt/victorops/nagios_plugin" +const VictorOps_LOG_DIR = "/var/log/victorops" +const VictorOps_QUEUE_DIR = "/var/nagios" +const VictorOps_PROTOCOL = "https" +const VictorOps_ALERT_HOST = "alert.victorops.com" +const VictorOps_ALERT_URL = "/nagios" +const VictorOps_COMMAND_URL = "/nagiosCmds" +const VictorOps_STATUS_URL = "/nagiosStatus" +const VictorOps_PING_URL = "/ping" + + +object NotificationCommand "victorops-notification" { + import "plugin-notification-command" + + command = VictorOps_PLUGIN_HOME + "/bin/enqueue_alert >> " + VictorOps_LOG_DIR + "/enqueue_alerts.log 2>&1" + + env = { + ICINGA_COMMANDFILE = IcingaCmdPipe + ICINGA_STATUSDATAFILE = IcingaStatusFile + ICINGA_TIMET = "$icinga.timet$" + ICINGA_NOTIFICATIONTYPE = "$notification.type$" + ICINGA_HOSTNAME = "$host.name$" + ICINGA_HOSTALIAS = "$host.display_name$" + ICINGA_HOSTADDRESS = "$address$" + ICINGA_HOSTSTATE = "$host.state$" + ICINGA_LASTHOSTSTATECHANGE = "$host.last_state_change$" + ICINGA_LONGDATETIME = "$icinga.long_date_time$" + ICINGA_NOTIFICATIONAUTHOR = "$notification.author$" + ICINGA_NOTIFICATIONCOMMENT = "$notification.comment$" + ICINGA_HOSTDISPLAYNAME = "$host.display_name$" + ICINGA_CONTACTGROUPNAME = "$user.name$" + + ICINGA__CONTACTVO_ORGANIZATION_ID = VictorOps_ORGANIZATION_ID + ICINGA__CONTACTVO_ORGANIZATION_KEY = VictorOps_ORGANIZATION_KEY + ICINGA__CONTACTVO_MONITOR_NAME = VictorOps_MONITOR_NAME + ICINGA__CONTACTVO_CONTACTEMAIL = VictorOps_CONTACTEMAIL + ICINGA__CONTACTVO_MAX_SEND_DELAY = VictorOps_MAX_SEND_DELAY + ICINGA__CONTACTVO_PLUGIN_HOME = VictorOps_PLUGIN_HOME + ICINGA__CONTACTVO_LOG_DIR = VictorOps_LOG_DIR + ICINGA__CONTACTVO_QUEUE_DIR = VictorOps_QUEUE_DIR + ICINGA__CONTACTVO_PROTOCOL = VictorOps_PROTOCOL + ICINGA__CONTACTVO_ALERT_HOST = VictorOps_ALERT_HOST + ICINGA__CONTACTVO_ALERT_URL = VictorOps_ALERT_URL + ICINGA__CONTACTVO_COMMAND_URL = VictorOps_COMMAND_URL + ICINGA__CONTACTVO_STATUS_URL = VictorOps_STATUS_URL + ICINGA__CONTACTVO_PING_URL = VictorOps_PING_URL + } +} + +object NotificationCommand "victorops-svc-notification" { + import "victorops-notification" + + env += { + ICINGA_SERVICEDESC = "$service.name$" + ICINGA_SERVICEDISPLAYNAME = "$service.display_name$" + ICINGA_SERVICEOUTPUT = "$service.output$" + ICINGA_SERVICESTATE = "$service.state$" + ICINGA_LASTSERVICESTATECHANGE = "$service.last_state_change$" + } +} + +object User "VictorOps" { + import "generic-user" + display_name = "VictorOps" +} + +template Notification "notify-victorops" { + users = [ "VictorOps" ] + types = [ Problem, Acknowledgement, Recovery ] + command = "victorops-notification" + period = "24x7" +} + +template Notification "notify-victorops-host" { + import "notify-victorops" + states = [ Up, Down ] +} + +template Notification "notify-victorops-service" { + import "notify-victorops" + states = [ Warning, Unknown, Critical, OK ] + command = "victorops-svc-notification" +} + +template CheckCommand "check-victorops" { + import "plugin-check-command" + + env = { + MONITOR_SW = "ICINGA2" + ICINGA_COMMANDFILE = IcingaCmdPipe + ICINGA_STATUSDATAFILE = IcingaStatusFile + ICINGA_HOSTNAME = NodeName + + ICINGA__SERVICEVO_ORGANIZATION_ID = VictorOps_ORGANIZATION_ID + ICINGA__SERVICEVO_ORGANIZATION_KEY = VictorOps_ORGANIZATION_KEY + ICINGA__SERVICEVO_MONITOR_NAME = VictorOps_MONITOR_NAME + ICINGA__SERVICEVO_CONTACTEMAIL = VictorOps_CONTACTEMAIL + ICINGA__SERVICEVO_MAX_SEND_DELAY = VictorOps_MAX_SEND_DELAY + ICINGA__SERVICEVO_PLUGIN_HOME = VictorOps_PLUGIN_HOME + ICINGA__SERVICEVO_LOG_DIR = VictorOps_LOG_DIR + ICINGA__SERVICEVO_QUEUE_DIR = VictorOps_QUEUE_DIR + ICINGA__SERVICEVO_PROTOCOL = VictorOps_PROTOCOL + ICINGA__SERVICEVO_ALERT_HOST = VictorOps_ALERT_HOST + ICINGA__SERVICEVO_ALERT_URL = VictorOps_ALERT_URL + ICINGA__SERVICEVO_COMMAND_URL = VictorOps_COMMAND_URL + ICINGA__SERVICEVO_STATUS_URL = VictorOps_STATUS_URL + ICINGA__SERVICEVO_PING_URL = VictorOps_PING_URL + } +} + +object CheckCommand "check-victorops-heartbeat" { + import "check-victorops" + command = VictorOps_PLUGIN_HOME + "/bin/do_victorops_heartbeat" +} + +object CheckCommand "check-victorops-forwarder" { + import "check-victorops" + command = VictorOps_PLUGIN_HOME + "/bin/check_victorops_forwarder 2>&1" +} + +object CheckCommand "check-victorops-commands" { + import "check-victorops" + command = VictorOps_PLUGIN_HOME + "/bin/check_victorops_cmds" +} + +template Service "victorops-service" { + host_name = NodeName + check_interval = 60s + retry_interval = 60s +} + +object Service "victorops-heartbeat" { + import "victorops-service" + + display_name = "VictorOps Heartbeat" + check_command = "check-victorops-heartbeat" + check_interval = 300s +} + +object Service "victorops-forwarder" { + import "victorops-service" + + display_name = "VictorOps Alert Forwarder" + check_command = "check-victorops-forwarder" +} + +object Service "victorops-cmd-poll" { + import "victorops-service" + + display_name = "VictorOps Command Poll" + check_command = "check-victorops-commands" + enable_active_checks = VictorOps_enable_cmd_poll +} + +#------------------------------------------------------------------------------------- +# Alert if the alert forwarder service is not running. +# Because it's not running, we can only alert via email. +#------------------------------------------------------------------------------------- +object NotificationCommand "victorops-forwarder-fail-notification" { + import "victorops-notification" + command = printf_executable + " '%b' '\\n$icinga.date$ $icinga.time$\\n\\n$notification.type$: $host.name$/$service.name$ is $service.state$\\n\\n' | " + mail_executable + " -s '$notification.type$: $host.name$/$service.name$ is $service.state$' " + VictorOps_CONTACTEMAIL + " failsafe+" + VictorOps_ORGANIZATION_ID + "@victorops.net >> " + VictorOps_LOG_DIR + "/forwarder_fail.out 2>&1" +} + +apply Notification "notify-forwarder-fail" to Service { + import "notify-victorops" + command = "victorops-forwarder-fail-notification" + states = [ Warning, Unknown, Critical, OK ] + assign where match("victorops-forwarder", service.name) +} + +object CheckCommand "manual-send-status" { + import "check-victorops" + command = VictorOps_PLUGIN_HOME + "/bin/send_status -m" +}