mongodb-backups: use systemd timers for backups

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/requirements.yml

@@ -23,3 +23,7 @@
 - name: swap-file
   src: git@github.com:status-im/infra-role-swap-file.git
   scm: git
+
+- name: systemd-timer
+  src: git@github.com:status-im/infra-role-systemd-timer.git
+  scm: git

ansible/roles/mongodb-backups/README.md

@@ -1,3 +1,31 @@
 # Description
 
 This role configures a cron job for making MongoDB dumps and uploading them to S3.
+
+# Management
+
+You can see existing dump/backup jobs using `systemctl`
+```
+admin@node-01.us-east-1a.db.prod:~ % sudo systemctl list-timers '*-mongod.timer'
+NEXT                         LEFT     LAST PASSED UNIT                ACTIVATES
+Wed 2020-04-01 00:00:00 UTC  15h left n/a  n/a    backup-mongod.timer backup-mongod.service
+Wed 2020-04-01 00:00:00 UTC  15h left n/a  n/a    dump-mongod.timer   dump-mongod.service
+
+2 timers listed.
+```
+And run one by hand:
+```
+admin@node-01.us-east-1a.db.prod:~ % sudo systemctl start dump-mongod.service
+admin@node-01.us-east-1a.db.prod:~ % sudo journalctl -fu dump-mongod.service
+-- Logs begin at Sun 2019-11-03 12:25:35 UTC. --
+systemd[1]: Starting MongoDB dump job...
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.923+0000        dumping up to 3 collections in parallel
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.926+0000        writing prod-dap-ps.dappsmetadatas to
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.927+0000        writing prod-dap-ps.dappsimages to
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.927+0000        writing prod-dap-ps.media to
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.992+0000        done dumping prod-dap-ps.media (78 documents)
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.993+0000        done dumping prod-dap-ps.dappsimages (119 documents)
+prod_dap_ps_dump.sh[17215]: 2020-03-31T08:55:55.994+0000        done dumping prod-dap-ps.dappsmetadatas (149 documents)
+prod_dap_ps_dump.sh[17215]: tar: Removing leading `/' from member names
+systemd[1]: Started MongoDB dump job.
+```

ansible/roles/mongodb-backups/defaults/main.yml

@@ -4,6 +4,8 @@ mongodb_backup_path: '/var/tmp/backups/mongodb'
 mongodb_backup_file_prefix: '{{ mongodb_backup_db_name | replace("-", "_") }}'
 mongodb_backup_script: '/var/lib/backups/{{ mongodb_backup_db_name | replace("-", "_") }}_dump.sh'
 mongodb_backup_frequency: daily
+mongodb_backup_timeout: 120
+mongodb_backup_user: root
 
 # db auth
 mongodb_backup_db_name: ~

ansible/roles/mongodb-backups/tasks/main.yml

@@ -10,39 +10,29 @@
     - '{{ mongodb_backup_path }}'
 
 - name: MongoDB | Create DB dump script
-  copy:
+  template:
+    src: 'dump.sh.j2'
     dest: '{{ mongodb_backup_script }}'
-    content: |
-      #!/usr/bin/env bash
-      TSTAMP=$(date -u +%Y%m%d%H%M%S)
-      BKP_DIR={{ mongodb_backup_path }}
-      BPK_NAME="{{ mongodb_backup_file_prefix }}_dump_${TSTAMP}"
-      mongodump \
-        --verbose \
-        --host={{ mongodb_backup_db_addr | mandatory }} \
-        --port={{ mongodb_backup_db_port | mandatory }} \
-        --username={{ mongodb_backup_db_user | mandatory }} \
-        --password={{ mongodb_backup_db_pass | mandatory }} \
-        --db={{ mongodb_backup_db_name | mandatory }} \
-        --authenticationDatabase={{ mongodb_backup_db_name | mandatory }} \
-        --out="${BKP_DIR}/${BPK_NAME}"
-      tar czf "${BKP_DIR}/${BPK_NAME}.tar.gz" "${BKP_DIR}/${BPK_NAME}"
-      rm -fr "${BKP_DIR}/${BPK_NAME}"
     group: adm
     mode: 0750
 
-- name: MongoDB | Configure DB dump cron job
-  cron:
-    name: MongoDB Dump
-    special_time: '{{ mongodb_backup_frequency }}'
-    user: root
-    job: '{{ mongodb_backup_script }}'
+- name: MongoDB | Configure DB dump timer
+  include_role: name=systemd-timer
+  vars:
+    systemd_timer_name: dump-mongod
+    systemd_timer_description: 'MongoDB dump job'
+    systemd_timer_user: '{{ mongodb_backup_user }}'
+    systemd_timer_frequency: '{{ mongodb_backup_frequency | default("daily") }}'
+    systemd_timer_timeout_sec: '{{ mongodb_backup_timeout }}'
+    systemd_timer_requires_extra: 'mongod.service'
+    systemd_timer_script_path: '{{ mongodb_backup_script }}'
+    systemd_timer_start_on_creation: false
 
 - name: MongoDB | Configure S3 upload cron job
   include_role:
     name: s3cmd-backup
   vars:
-    backup_name: mongodb-backups
+    backup_name: mongod
     backup_number: 1
     backup_hour: 4
     backup_day: '*'

ansible/roles/mongodb-backups/templates/dump.sh.j2

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+TSTAMP=$(date -u +%Y%m%d%H%M%S)
+BKP_DIR={{ mongodb_backup_path }}
+BPK_NAME="{{ mongodb_backup_file_prefix }}_dump_${TSTAMP}"
+mongodump \
+    --verbose \
+    --host={{ mongodb_backup_db_addr | mandatory }} \
+    --port={{ mongodb_backup_db_port | mandatory }} \
+    --username={{ mongodb_backup_db_user | mandatory }} \
+    --password={{ mongodb_backup_db_pass | mandatory }} \
+    --db={{ mongodb_backup_db_name | mandatory }} \
+    --authenticationDatabase={{ mongodb_backup_db_name | mandatory }} \
+    --out="${BKP_DIR}/${BPK_NAME}"
+tar czf "${BKP_DIR}/${BPK_NAME}.tar.gz" "${BKP_DIR}/${BPK_NAME}"
+rm -fr "${BKP_DIR}/${BPK_NAME}"