feat: publish debug csv to github repo

defaults/main.yml

@@ -4,9 +4,14 @@ nimbus_eth1_benchmark_type: 'short'
 nimbus_eth1_service_name: 'nimbus-eth1-{{ nimbus_eth1_network }}-{{ nimbus_eth1_repo_branch }}-{{ nimbus_eth1_benchmark_type }}-benchmark'
 nimbus_eth1_service_path: '/home/data/{{ nimbus_eth1_service_name }}'
 nimbus_eth1_service_user: 'nimbus'
+nimbus_eth1_service_user_ssh_dir: '/home/{{ nimbus_eth1_service_user }}/.ssh'
+nimbus_eth1_service_user_ssh_config: '{{ nimbus_eth1_service_user_ssh_dir }}/config'
 nimbus_eth1_service_group: 'staff'
 nimbus_eth1_template_db: '/data/template-db'
-nimbus_eth1_db_dir: 'data/shared_mainnet_0/nimbus/aristo'
+nimbus_eth1_db_dir: '{{ nimbus_eth1_service_path }}/data/shared_mainnet_0/nimbus/aristo'
+nimbus_eth1_publish_script: '/data/publish-metrics-github.sh'
+nimbus_eth1_benchmark_git_repo_dir: '/data/nimbus-eth1-benchmarks'
+nimbus_eth1_benchmarks_git_repo_url: 'git@github.com:status-im/nimbus-eth1-benchmarks.git'
 
 # Paths
 nimbus_eth1_data_path: '{{ nimbus_eth1_service_path }}/data'
@@ -53,8 +58,7 @@ nimbus_eth1_jwt_secret_path: '{{ nimbus_eth1_data_path }}/jwt.hex'
 nimbus_eth1_build_timer_enabled: true
 nimbus_eth1_build_service_name: 'build-{{ nimbus_eth1_service_name }}'
 nimbus_eth1_build_script_path: '{{ nimbus_eth1_service_path }}/build.sh'
-nimbus_eth1_build_frequency: 'daily'
-nimbus_eth1_build_timer_timeout: 3600
+nimbus_eth1_build_timer_timeout: 36000
 nimbus_eth1_build_days_kept: 3
 # FIXME: remove -d:disableMarchNative when https://github.com/bitcoin-core/secp256k1/issues/1623 is fixed
 nimbus_eth1_build_nim_flags: ['-d:chronicles_colors=none','-d:disableMarchNative']
@@ -63,8 +67,5 @@ nimbus_eth1_build_nim_flags: ['-d:chronicles_colors=none','-d:disableMarchNative
 nimbus_eth1_repo_branch: 'master'
 nimbus_eth1_repo_url: 'https://github.com/status-im/nimbus-eth1'
 nimbus_eth1_repo_path: '{{ nimbus_eth1_service_path }}/repo'
-
-# Consul service definition settings
-nimbus_eth1_consul_service_name: 'nimbus-eth1'
-nimbus_eth1_consul_service_file_name: '{{ nimbus_eth1_service_name | replace("-", "_") }}'
-nimbus_eth1_consul_metrics_service_name: '{{ nimbus_eth1_consul_service_name }}-metrics'
+nimbus_eth1_benchmarks_ssh_key_file: '{{ nimbus_eth1_service_user_ssh_dir }}/id_rsa'
+nimbus_eth1_benchmarks_repo_ssh_key: '{{lookup("vault", "nimbus/eth1/benchmark", field="github_webhook_ssh_key", override=True)}}'

files/ssh_config

@@ -0,0 +1,4 @@
+Host github.com/status-im/nimbus-eth1-benchmarks
+        Hostname github.com
+        StrictHostKeyChecking accept-new
+        IdentityFile=/home/nimbus/.ssh/id_rsa

tasks/build.yml

@@ -31,7 +31,6 @@
     systemd_timer_start_on_creation: '{{ not nimbus_eth1_bin.stat.exists }}'
     systemd_timer_consul_warning: true
     systemd_timer_enabled: '{{ nimbus_eth1_build_timer_enabled }}'
-    systemd_timer_frequency: '{{ nimbus_eth1_build_frequency }}'
     systemd_timer_timeout_sec: '{{ nimbus_eth1_build_timer_timeout }}'
     systemd_timer_work_dir: '{{ nimbus_eth1_service_path }}'
     systemd_timer_script_path: '{{ nimbus_eth1_build_script_path }}'

tasks/config.yml

@@ -10,6 +10,60 @@
     - '{{ nimbus_eth1_service_path }}'
     - '{{ nimbus_eth1_data_path }}'
 
+- name: Create SSH config directory
+  file:
+    path:  '{{ nimbus_eth1_service_user_ssh_dir }}'
+    owner: '{{ nimbus_eth1_service_user }}'
+    group: '{{ nimbus_eth1_service_group }}'
+    state: 'directory'
+    mode: 0750
+  when: nimbus_eth1_service_user_ssh_dir is defined
+
+- name: Check if SSH config exists
+  command: grep -F "StrictHostKeyChecking no" {{ nimbus_eth1_service_user_ssh_config }}
+  register: ssh_config_check
+  failed_when: false
+  changed_when: false
+  check_mode: no
+
+- name: Disable strict host key checking for GitHub
+  copy:
+    src:   'ssh_config'
+    dest:  '{{ nimbus_eth1_service_user_ssh_config }}'
+    owner: '{{ nimbus_eth1_service_user }}'
+    group: '{{ nimbus_eth1_service_group }}'
+    mode:  0400
+  when: ssh_config_check.rc != 0
+
+- name: Check if SSH key exists
+  command: grep -F "{{ nimbus_eth1_benchmarks_repo_ssh_key | regex_escape() }}" {{ nimbus_eth1_benchmarks_ssh_key_file }}
+  register: ssh_key_check
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  when: nimbus_eth1_benchmarks_ssh_key_file is defined
+
+- name: Append SSH private key
+  copy:
+    dest: '{{ nimbus_eth1_benchmarks_ssh_key_file }}'
+    content: |
+      {{ nimbus_eth1_benchmarks_repo_ssh_key }}
+    owner: '{{ nimbus_eth1_service_user }}'
+    group: '{{ nimbus_eth1_service_group }}'
+    mode: 0400
+  when:
+    - nimbus_eth1_benchmarks_ssh_key_file is defined
+#    - ssh_key_check.rc != 0
+
+- name: Clone git repo with existing benchmarks
+  git:
+    repo:    '{{ nimbus_eth1_benchmarks_git_repo_url }}'
+    dest:    '{{ nimbus_eth1_benchmark_git_repo_dir }}'
+    key_file: '{{ nimbus_eth1_benchmarks_ssh_key_file }}'
+    accept_hostkey: true
+    update: true
+    force: true
+
 - name: Create JSON RPC wrapper script
   template:
     src:    'rpc.sh.j2'

tasks/service.yml

@@ -28,9 +28,39 @@
     dest: '/etc/systemd/system/{{ nimbus_eth1_service_name }}.service'
     mode: 0644
 
+- name: Create metrics github publish script
+  template:
+    src: 'publish-metrics-github.sh.j2'
+    dest: '{{ nimbus_eth1_publish_script }}'
+    owner: '{{ nimbus_eth1_service_user }}'
+    group: '{{ nimbus_eth1_service_group }}'
+    mode: 0775
+
 - name: Reload systemctl
   command: systemctl daemon-reload
 
+- name: Clean up existing DB
+  block:
+    - name: Remove existing DB directory
+      file:
+        path: "{{ nimbus_eth1_db_dir }}"
+        state: absent
+      become: true
+
+- name: Create DB directory
+  file:
+    path: "{{ nimbus_eth1_db_dir }}"
+    state: directory
+    mode: '0755'
+  become: true
+
+- name: Copy template DB contents
+  copy:
+    src: "{{ nimbus_eth1_template_db }}/"
+    dest: "{{ nimbus_eth1_db_dir }}"
+    remote_src: true
+  become: true
+
 - name: Restart the service
   service:
     name: '{{ nimbus_eth1_service_name }}.service'

templates/build.sh.j2

@@ -28,22 +28,11 @@ function buildBinaries() {
     find build -mtime +{{ nimbus_eth1_build_days_kept }} -exec rm '{}' \+
 }
 
-function copyTemplateDB() {
-    echo " >>> Clean up existing db folder..."
-    rm -vrf "${ETH1_DB_DIR}"
-    echo " >>> Preparing path..."
-    mkdir -p "${ETH1_DB_DIR}"
-    echo " >>> Copy Template Db..."
-    cp -rf "${TEMPLATE_DB}/*" "${ETH1_DB_DIR}"
-}
-
 #-------------------------------------------------------------------------------
 
 BRANCH="{{ nimbus_eth1_repo_branch }}"
 SERVICE="{{ nimbus_eth1_service_name }}.service"
 BUILD_DIR="{{ nimbus_eth1_repo_path }}"
-TEMPLATE_DB="{{ nimbus_eth1_template_db }}"
-ETH1_DB_DIR="{{ nimbus_eth1_db_dir }}"
 
 if [[ "${USER}" != "{{ nimbus_eth1_service_user }}" ]]; then
     echo "Incorrect user: ${USER}" >&2
@@ -66,7 +55,6 @@ COMMIT=$(git rev-parse --short=8 HEAD)
 
 echo " >>> Building binaries..."
 buildBinaries
-copyTemplateDB
 
 # Avoid failure on first Ansible run due to missing service.
 if [[ $(systemctl is-active "${SERVICE}" || true) == "inactive" ]]; then

templates/nimbus-eth1-short-benchmark.service.j2

@@ -16,7 +16,8 @@ ExecStart={{ nimbus_eth1_repo_path }}/build/nimbus_execution_client import \
     --data-dir='{{ nimbus_eth1_data_path }}/{{ nimbus_eth1_data_folder }}' \
     --era1-dir={{ nimbus_eth1_era1_dir | mandatory }} \
     --era-dir={{ nimbus_eth1_era_dir | mandatory }} \
-    --metrics --debug-csv-stats=/data/{{ benchmark_file_name }}
+    --debug-csv-stats=/data/{{ benchmark_file_name }}
+ExecStopPost={{ nimbus_eth1_publish_script }}
 
 [Install]
 WantedBy=multi-user.target

templates/publish-metrics-github.sh.j2

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# vim: ft=sh
+set -e
+
+BENCHMARK_FILE_NAME="{{ benchmark_file_name }}"
+BENCHMARK_GIT_REPO="{{ nimbus_eth1_benchmark_git_repo_dir }}"
+
+function fetchChanges() {
+    echo " >>> Fetching latest changes from nimbus-eth1-benchmarks repo..."
+    # We cannot use "git pull" in here, because history may be changed upstream
+    cd "${BENCHMARK_GIT_REPO}"
+    git fetch
+    git reset --hard "origin/main"
+    cd -
+}
+
+function moveMetricsCSVToRepo() {
+  cd /data
+  echo " >>> Moving benchmark csv to checked out git repo..."
+  mv "./${BENCHMARK_FILE_NAME}" "${BENCHMARK_GIT_REPO}/${BENCHMARK_FILE_NAME}"
+}
+
+function pushChangesToRepo() {
+  cd "${BENCHMARK_GIT_REPO}"
+  git add "./${BENCHMARK_FILE_NAME}"
+  git commit -m "benchmark: publish metrics"
+  git push
+  echo " >>> Pushed benchmark csv to nimbus-eth1-benchmarks repo..."
+}
+
+fetchChanges
+moveMetricsCSVToRepo
+pushChangesToRepo
+
+