consul: add health.sh to check replica status

Signed-off-by: Jakub Sokołowski <jakub@status.im>

defaults/main.yml

@@ -18,6 +18,7 @@ postgres_ha_cont_tmp_size: '256K'
 postgres_ha_admin_user: 'postgres'
 postgres_ha_admin_pass: 'changeMeIfYouCare'
 # Replication Auth
+postgres_ha_replica_enabled: true
 postgres_ha_replica_user: 'replicator'
 postgres_ha_replica_pass: 'changeMeIfYouCare'
 postgres_ha_replica_slot: 'main'

tasks/consul.yml

@@ -9,7 +9,13 @@
         tags: ['postgres', 'database']
         port: '{{ postgres_ha_cont_port }}'
         checks:
-          - id:   '{{ postgres_ha_service_name }}-status'
-            name: 'PostgreSQL DB Healthcheck'
-            type: 'tcp'
-            tcp:  'localhost:{{ postgres_ha_cont_port }}'
+          - id:       '{{ postgres_ha_service_name }}-status'
+            name:     'PostgreSQL DB Healthcheck'
+            type:     'tcp'
+            tcp:      'localhost:{{ postgres_ha_cont_port }}'
+
+          - id:       '{{ postgres_ha_service_name }}-mirror'
+            name:     'PostgreSQL Replica Healthcheck'
+            disabled: '{{ not postgres_ha_replica_enabled }}'
+            type:     'script'
+            script:   '{{ postgres_ha_service_path }}/health.sh'

tasks/main.yml

@@ -8,7 +8,7 @@
   when: postgres_ha_is_master
 
 - import_tasks: users.yml
-  when: postgres_ha_is_master
+  when: postgres_ha_is_master and postgres_ha_replica_enabled
 
 - import_tasks: replica.yml
   when: not postgres_ha_is_master

templates/scripts/health.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# vim: set ft=sh:
+set -Eeo pipefail
+
+{% if postgres_ha_is_master %}
+STATUS_QUERY='SELECT * FROM pg_stat_replication'
+{% else %}
+STATUS_QUERY='SELECT * FROM pg_stat_wal_receiver'
+{% endif %}
+REPLICA_STATUS=$(
+    "{{ postgres_ha_service_path }}/admin.sh" -x -c "${STATUS_QUERY}"
+)
+
+function fail() {
+    echo "ERROR: Replication broken!"
+    echo
+    echo "${REPLICA_STATUS}";
+    exit 2;
+}
+
+{% if postgres_ha_is_master %}
+REGEX='state +| streaming'
+{% else %}
+REGEX='status +| streaming'
+{% endif %}
+[[ ! "${REPLICA_STATUS}" =~ ${REGEX} ]] && fail
+
+echo "Replication healthy."
+echo
+echo "${REPLICA_STATUS}" | grep -v -e '_lsn' -e '_lag'