From 56e3f1d5cbcc8cb2d8a64907d55e845f9a2fb496 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= <jakub@status.im>
Date: Wed, 16 Sep 2020 12:50:46 +0200
Subject: [PATCH] add network-debug-script to figure out host issues
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Jakub Sokołowski <jakub@status.im>
---
 ansible/nimbus.yml                            |  5 +++-
 .../network-debug-script/defaults/main.yml    |  7 ++++++
 .../roles/network-debug-script/tasks/main.yml | 24 +++++++++++++++++++
 .../roles/network-debug-script/templates/;    | 21 ++++++++++++++++
 .../templates/debug_script.sh                 | 19 +++++++++++++++
 5 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/network-debug-script/defaults/main.yml
 create mode 100644 ansible/roles/network-debug-script/tasks/main.yml
 create mode 100644 ansible/roles/network-debug-script/templates/;
 create mode 100644 ansible/roles/network-debug-script/templates/debug_script.sh

diff --git a/ansible/nimbus.yml b/ansible/nimbus.yml
index f3d5da5..43e05fa 100644
--- a/ansible/nimbus.yml
+++ b/ansible/nimbus.yml
@@ -32,6 +32,8 @@
         beacon_node_cont_name: 'beacon-node-{{ beacon_node_network }}'
         beacon_node_shared_testnet_cron: true
         cont_restart: true
+  roles:
+    - { role: 'network-debug-script', tags: 'network-debug-script' }
 
 - name: Configure shared testnet2
   hosts:
@@ -51,4 +53,5 @@
         beacon_node_cont_name: 'beacon-node-{{ beacon_node_network }}'
         beacon_node_shared_testnet_cron: true
         cont_restart: true
-
+  roles:
+    - { role: 'network-debug-script', tags: 'network-debug-script' }
diff --git a/ansible/roles/network-debug-script/defaults/main.yml b/ansible/roles/network-debug-script/defaults/main.yml
new file mode 100644
index 0000000..b0728b6
--- /dev/null
+++ b/ansible/roles/network-debug-script/defaults/main.yml
@@ -0,0 +1,7 @@
+---
+net_debug_user: 'root'
+net_debug_frequency: '*:0/5' # every 5 minutes
+net_debug_timeout: 30
+net_debug_logs_dir: '/var/log/net_debug'
+net_debug_keep_logs: 8640 # 12 hours
+net_debug_script_path: '/usr/local/bin/network_debug_script.sh'
diff --git a/ansible/roles/network-debug-script/tasks/main.yml b/ansible/roles/network-debug-script/tasks/main.yml
new file mode 100644
index 0000000..5e7c660
--- /dev/null
+++ b/ansible/roles/network-debug-script/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Create logs directory
+  file:
+    path: '{{ net_debug_logs_dir }}'
+    state: directory
+
+- name: Copy over the debug script
+  template:
+    src: debug_script.sh
+    dest: '{{ net_debug_script_path }}'
+    mode: 0755
+
+- name: Set systemd timer for debug script
+  include_role: name=systemd-timer
+  vars:
+    systemd_timer_name: 'network-debug-script'
+    systemd_timer_description: 'Network Debug Script'
+    systemd_timer_user: '{{ net_debug_user }}'
+    systemd_timer_frequency: '{{ net_debug_frequency }}'
+    systemd_timer_timeout_sec: '{{ net_debug_timeout}}'
+    systemd_timer_work_dir: '{{ net_debug_logs_dir }}'
+    systemd_timer_requires_extra: 'network.target'
+    systemd_timer_script_path: '{{ net_debug_script_path }}'
+    systemd_timer_start_on_creation: false
diff --git a/ansible/roles/network-debug-script/templates/; b/ansible/roles/network-debug-script/templates/;
new file mode 100644
index 0000000..7145f1c
--- /dev/null
+++ b/ansible/roles/network-debug-script/templates/;
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+function get_network_state() {
+    set -x
+    ip route
+    echo
+    ip --brief address show
+    echo
+    docker network list
+}
+
+MAX_KEPT={{ net_debug_keep_logs }}
+LOG_DIR="{{ net_debug_logs_dir }}"
+TSTAMP=$(date -u +%Y%m%d%H%M%S)
+
+# Save network state
+get_network_state > 2>&1 "${LOG_DIR}/${TSTAMP}.log"
+
+# Clean old states
+OLD_LOGS=$(ls -Art ${LOG_DIR} | head -n -${MAX_KEPT})
+rm -vf ${OLD_LOGS}
diff --git a/ansible/roles/network-debug-script/templates/debug_script.sh b/ansible/roles/network-debug-script/templates/debug_script.sh
new file mode 100644
index 0000000..47e32c2
--- /dev/null
+++ b/ansible/roles/network-debug-script/templates/debug_script.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+function get_network_state() {
+    set -x
+    ip route
+    ip --brief address show
+    docker network list
+}
+
+MAX_KEPT={{ net_debug_keep_logs }}
+LOG_DIR="{{ net_debug_logs_dir }}"
+TSTAMP=$(date -u +%Y%m%d%H%M%S)
+
+# Save network state
+get_network_state > "${LOG_DIR}/${TSTAMP}.log" 2>&1
+
+# Clean old states
+OLD_LOGS=$(ls -Art ${LOG_DIR} | head -n -${MAX_KEPT})
+rm -vf ${OLD_LOGS}