From 93f7758338a2b0e879c82e28fe2cd4d0f54afc87 Mon Sep 17 00:00:00 2001 From: Siddarth Kumar Date: Tue, 26 Nov 2024 18:05:16 +0530 Subject: [PATCH] feat: publish debug csv to github repo --- defaults/main.yml | 17 +++--- files/ssh_config | 4 ++ tasks/build.yml | 1 - tasks/config.yml | 54 +++++++++++++++++++ tasks/service.yml | 30 +++++++++++ templates/build.sh.j2 | 12 ----- .../nimbus-eth1-short-benchmark.service.j2 | 3 +- templates/publish-metrics-github.sh.j2 | 35 ++++++++++++ 8 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 files/ssh_config create mode 100644 templates/publish-metrics-github.sh.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 97f679d..6a7600b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -4,9 +4,14 @@ nimbus_eth1_benchmark_type: 'short' nimbus_eth1_service_name: 'nimbus-eth1-{{ nimbus_eth1_network }}-{{ nimbus_eth1_repo_branch }}-{{ nimbus_eth1_benchmark_type }}-benchmark' nimbus_eth1_service_path: '/home/data/{{ nimbus_eth1_service_name }}' nimbus_eth1_service_user: 'nimbus' +nimbus_eth1_service_user_ssh_dir: '/home/{{ nimbus_eth1_service_user }}/.ssh' +nimbus_eth1_service_user_ssh_config: '{{ nimbus_eth1_service_user_ssh_dir }}/config' nimbus_eth1_service_group: 'staff' nimbus_eth1_template_db: '/data/template-db' -nimbus_eth1_db_dir: 'data/shared_mainnet_0/nimbus/aristo' +nimbus_eth1_db_dir: '{{ nimbus_eth1_service_path }}/data/shared_mainnet_0/nimbus/aristo' +nimbus_eth1_publish_script: '/data/publish-metrics-github.sh' +nimbus_eth1_benchmark_git_repo_dir: '/data/nimbus-eth1-benchmarks' +nimbus_eth1_benchmarks_git_repo_url: 'git@github.com:status-im/nimbus-eth1-benchmarks.git' # Paths nimbus_eth1_data_path: '{{ nimbus_eth1_service_path }}/data' @@ -53,8 +58,7 @@ nimbus_eth1_jwt_secret_path: '{{ nimbus_eth1_data_path }}/jwt.hex' nimbus_eth1_build_timer_enabled: true nimbus_eth1_build_service_name: 'build-{{ nimbus_eth1_service_name }}' nimbus_eth1_build_script_path: '{{ nimbus_eth1_service_path }}/build.sh' -nimbus_eth1_build_frequency: 'daily' -nimbus_eth1_build_timer_timeout: 3600 +nimbus_eth1_build_timer_timeout: 36000 nimbus_eth1_build_days_kept: 3 # FIXME: remove -d:disableMarchNative when https://github.com/bitcoin-core/secp256k1/issues/1623 is fixed nimbus_eth1_build_nim_flags: ['-d:chronicles_colors=none','-d:disableMarchNative'] @@ -63,8 +67,5 @@ nimbus_eth1_build_nim_flags: ['-d:chronicles_colors=none','-d:disableMarchNative nimbus_eth1_repo_branch: 'master' nimbus_eth1_repo_url: 'https://github.com/status-im/nimbus-eth1' nimbus_eth1_repo_path: '{{ nimbus_eth1_service_path }}/repo' - -# Consul service definition settings -nimbus_eth1_consul_service_name: 'nimbus-eth1' -nimbus_eth1_consul_service_file_name: '{{ nimbus_eth1_service_name | replace("-", "_") }}' -nimbus_eth1_consul_metrics_service_name: '{{ nimbus_eth1_consul_service_name }}-metrics' +nimbus_eth1_benchmarks_ssh_key_file: '{{ nimbus_eth1_service_user_ssh_dir }}/id_rsa' +nimbus_eth1_benchmarks_repo_ssh_key: '{{lookup("vault", "nimbus/eth1/benchmark", field="github_webhook_ssh_key", override=True)}}' diff --git a/files/ssh_config b/files/ssh_config new file mode 100644 index 0000000..625b66d --- /dev/null +++ b/files/ssh_config @@ -0,0 +1,4 @@ +Host github.com/status-im/nimbus-eth1-benchmarks + Hostname github.com + StrictHostKeyChecking accept-new + IdentityFile=/home/nimbus/.ssh/id_rsa diff --git a/tasks/build.yml b/tasks/build.yml index afa5583..077c4b3 100644 --- a/tasks/build.yml +++ b/tasks/build.yml @@ -31,7 +31,6 @@ systemd_timer_start_on_creation: '{{ not nimbus_eth1_bin.stat.exists }}' systemd_timer_consul_warning: true systemd_timer_enabled: '{{ nimbus_eth1_build_timer_enabled }}' - systemd_timer_frequency: '{{ nimbus_eth1_build_frequency }}' systemd_timer_timeout_sec: '{{ nimbus_eth1_build_timer_timeout }}' systemd_timer_work_dir: '{{ nimbus_eth1_service_path }}' systemd_timer_script_path: '{{ nimbus_eth1_build_script_path }}' diff --git a/tasks/config.yml b/tasks/config.yml index 2b796f9..297f0d0 100644 --- a/tasks/config.yml +++ b/tasks/config.yml @@ -10,6 +10,60 @@ - '{{ nimbus_eth1_service_path }}' - '{{ nimbus_eth1_data_path }}' +- name: Create SSH config directory + file: + path: '{{ nimbus_eth1_service_user_ssh_dir }}' + owner: '{{ nimbus_eth1_service_user }}' + group: '{{ nimbus_eth1_service_group }}' + state: 'directory' + mode: 0750 + when: nimbus_eth1_service_user_ssh_dir is defined + +- name: Check if SSH config exists + command: grep -F "StrictHostKeyChecking no" {{ nimbus_eth1_service_user_ssh_config }} + register: ssh_config_check + failed_when: false + changed_when: false + check_mode: no + +- name: Disable strict host key checking for GitHub + copy: + src: 'ssh_config' + dest: '{{ nimbus_eth1_service_user_ssh_config }}' + owner: '{{ nimbus_eth1_service_user }}' + group: '{{ nimbus_eth1_service_group }}' + mode: 0400 + when: ssh_config_check.rc != 0 + +- name: Check if SSH key exists + command: grep -F "{{ nimbus_eth1_benchmarks_repo_ssh_key | regex_escape() }}" {{ nimbus_eth1_benchmarks_ssh_key_file }} + register: ssh_key_check + failed_when: false + changed_when: false + check_mode: no + when: nimbus_eth1_benchmarks_ssh_key_file is defined + +- name: Append SSH private key + copy: + dest: '{{ nimbus_eth1_benchmarks_ssh_key_file }}' + content: | + {{ nimbus_eth1_benchmarks_repo_ssh_key }} + owner: '{{ nimbus_eth1_service_user }}' + group: '{{ nimbus_eth1_service_group }}' + mode: 0400 + when: + - nimbus_eth1_benchmarks_ssh_key_file is defined +# - ssh_key_check.rc != 0 + +- name: Clone git repo with existing benchmarks + git: + repo: '{{ nimbus_eth1_benchmarks_git_repo_url }}' + dest: '{{ nimbus_eth1_benchmark_git_repo_dir }}' + key_file: '{{ nimbus_eth1_benchmarks_ssh_key_file }}' + accept_hostkey: true + update: true + force: true + - name: Create JSON RPC wrapper script template: src: 'rpc.sh.j2' diff --git a/tasks/service.yml b/tasks/service.yml index 00cf71b..1ed4363 100644 --- a/tasks/service.yml +++ b/tasks/service.yml @@ -28,9 +28,39 @@ dest: '/etc/systemd/system/{{ nimbus_eth1_service_name }}.service' mode: 0644 +- name: Create metrics github publish script + template: + src: 'publish-metrics-github.sh.j2' + dest: '{{ nimbus_eth1_publish_script }}' + owner: '{{ nimbus_eth1_service_user }}' + group: '{{ nimbus_eth1_service_group }}' + mode: 0775 + - name: Reload systemctl command: systemctl daemon-reload +- name: Clean up existing DB + block: + - name: Remove existing DB directory + file: + path: "{{ nimbus_eth1_db_dir }}" + state: absent + become: true + +- name: Create DB directory + file: + path: "{{ nimbus_eth1_db_dir }}" + state: directory + mode: '0755' + become: true + +- name: Copy template DB contents + copy: + src: "{{ nimbus_eth1_template_db }}/" + dest: "{{ nimbus_eth1_db_dir }}" + remote_src: true + become: true + - name: Restart the service service: name: '{{ nimbus_eth1_service_name }}.service' diff --git a/templates/build.sh.j2 b/templates/build.sh.j2 index 286e99e..e1a78e2 100644 --- a/templates/build.sh.j2 +++ b/templates/build.sh.j2 @@ -28,22 +28,11 @@ function buildBinaries() { find build -mtime +{{ nimbus_eth1_build_days_kept }} -exec rm '{}' \+ } -function copyTemplateDB() { - echo " >>> Clean up existing db folder..." - rm -vrf "${ETH1_DB_DIR}" - echo " >>> Preparing path..." - mkdir -p "${ETH1_DB_DIR}" - echo " >>> Copy Template Db..." - cp -rf "${TEMPLATE_DB}/*" "${ETH1_DB_DIR}" -} - #------------------------------------------------------------------------------- BRANCH="{{ nimbus_eth1_repo_branch }}" SERVICE="{{ nimbus_eth1_service_name }}.service" BUILD_DIR="{{ nimbus_eth1_repo_path }}" -TEMPLATE_DB="{{ nimbus_eth1_template_db }}" -ETH1_DB_DIR="{{ nimbus_eth1_db_dir }}" if [[ "${USER}" != "{{ nimbus_eth1_service_user }}" ]]; then echo "Incorrect user: ${USER}" >&2 @@ -66,7 +55,6 @@ COMMIT=$(git rev-parse --short=8 HEAD) echo " >>> Building binaries..." buildBinaries -copyTemplateDB # Avoid failure on first Ansible run due to missing service. if [[ $(systemctl is-active "${SERVICE}" || true) == "inactive" ]]; then diff --git a/templates/nimbus-eth1-short-benchmark.service.j2 b/templates/nimbus-eth1-short-benchmark.service.j2 index c6d5200..92c6c3d 100644 --- a/templates/nimbus-eth1-short-benchmark.service.j2 +++ b/templates/nimbus-eth1-short-benchmark.service.j2 @@ -16,7 +16,8 @@ ExecStart={{ nimbus_eth1_repo_path }}/build/nimbus_execution_client import \ --data-dir='{{ nimbus_eth1_data_path }}/{{ nimbus_eth1_data_folder }}' \ --era1-dir={{ nimbus_eth1_era1_dir | mandatory }} \ --era-dir={{ nimbus_eth1_era_dir | mandatory }} \ - --metrics --debug-csv-stats=/data/{{ benchmark_file_name }} + --debug-csv-stats=/data/{{ benchmark_file_name }} +ExecStopPost={{ nimbus_eth1_publish_script }} [Install] WantedBy=multi-user.target diff --git a/templates/publish-metrics-github.sh.j2 b/templates/publish-metrics-github.sh.j2 new file mode 100644 index 0000000..466f198 --- /dev/null +++ b/templates/publish-metrics-github.sh.j2 @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# vim: ft=sh +set -e + +BENCHMARK_FILE_NAME="{{ benchmark_file_name }}" +BENCHMARK_GIT_REPO="{{ nimbus_eth1_benchmark_git_repo_dir }}" + +function fetchChanges() { + echo " >>> Fetching latest changes from nimbus-eth1-benchmarks repo..." + # We cannot use "git pull" in here, because history may be changed upstream + cd "${BENCHMARK_GIT_REPO}" + git fetch + git reset --hard "origin/main" + cd - +} + +function moveMetricsCSVToRepo() { + cd /data + echo " >>> Moving benchmark csv to checked out git repo..." + mv "./${BENCHMARK_FILE_NAME}" "${BENCHMARK_GIT_REPO}/${BENCHMARK_FILE_NAME}" +} + +function pushChangesToRepo() { + cd "${BENCHMARK_GIT_REPO}" + git add "./${BENCHMARK_FILE_NAME}" + git commit -m "benchmark: publish metrics" + git push + echo " >>> Pushed benchmark csv to nimbus-eth1-benchmarks repo..." +} + +fetchChanges +moveMetricsCSVToRepo +pushChangesToRepo + +