add network-debug-script to figure out host issues

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/nimbus.yml

@@ -32,6 +32,8 @@
         beacon_node_cont_name: 'beacon-node-{{ beacon_node_network }}'
         beacon_node_shared_testnet_cron: true
         cont_restart: true
+  roles:
+    - { role: 'network-debug-script', tags: 'network-debug-script' }
 
 - name: Configure shared testnet2
   hosts:
@@ -51,4 +53,5 @@
         beacon_node_cont_name: 'beacon-node-{{ beacon_node_network }}'
         beacon_node_shared_testnet_cron: true
         cont_restart: true
-
+  roles:
+    - { role: 'network-debug-script', tags: 'network-debug-script' }

ansible/roles/network-debug-script/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+net_debug_user: 'root'
+net_debug_frequency: '*:0/5' # every 5 minutes
+net_debug_timeout: 30
+net_debug_logs_dir: '/var/log/net_debug'
+net_debug_keep_logs: 8640 # 12 hours
+net_debug_script_path: '/usr/local/bin/network_debug_script.sh'

ansible/roles/network-debug-script/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: Create logs directory
+  file:
+    path: '{{ net_debug_logs_dir }}'
+    state: directory
+
+- name: Copy over the debug script
+  template:
+    src: debug_script.sh
+    dest: '{{ net_debug_script_path }}'
+    mode: 0755
+
+- name: Set systemd timer for debug script
+  include_role: name=systemd-timer
+  vars:
+    systemd_timer_name: 'network-debug-script'
+    systemd_timer_description: 'Network Debug Script'
+    systemd_timer_user: '{{ net_debug_user }}'
+    systemd_timer_frequency: '{{ net_debug_frequency }}'
+    systemd_timer_timeout_sec: '{{ net_debug_timeout}}'
+    systemd_timer_work_dir: '{{ net_debug_logs_dir }}'
+    systemd_timer_requires_extra: 'network.target'
+    systemd_timer_script_path: '{{ net_debug_script_path }}'
+    systemd_timer_start_on_creation: false

ansible/roles/network-debug-script/templates/;

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+function get_network_state() {
+    set -x
+    ip route
+    echo
+    ip --brief address show
+    echo
+    docker network list
+}
+
+MAX_KEPT={{ net_debug_keep_logs }}
+LOG_DIR="{{ net_debug_logs_dir }}"
+TSTAMP=$(date -u +%Y%m%d%H%M%S)
+
+# Save network state
+get_network_state > 2>&1 "${LOG_DIR}/${TSTAMP}.log"
+
+# Clean old states
+OLD_LOGS=$(ls -Art ${LOG_DIR} | head -n -${MAX_KEPT})
+rm -vf ${OLD_LOGS}

ansible/roles/network-debug-script/templates/debug_script.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+function get_network_state() {
+    set -x
+    ip route
+    ip --brief address show
+    docker network list
+}
+
+MAX_KEPT={{ net_debug_keep_logs }}
+LOG_DIR="{{ net_debug_logs_dir }}"
+TSTAMP=$(date -u +%Y%m%d%H%M%S)
+
+# Save network state
+get_network_state > "${LOG_DIR}/${TSTAMP}.log" 2>&1
+
+# Clean old states
+OLD_LOGS=$(ls -Art ${LOG_DIR} | head -n -${MAX_KEPT})
+rm -vf ${OLD_LOGS}