From 82e76b9452da1aa3366a31b23bb04b5670a01e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 14 Sep 2023 18:30:43 +0200 Subject: [PATCH] add Holesky testnet fleet layout and configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/status-im/infra-nimbus/issues/152 Signed-off-by: Jakub Sokołowski --- ansible/group_vars/nimbus.holesky.yml | 81 ++++++++++++ ansible/holesky.yml | 50 +++++++ ansible/vars/layout/holesky.yml | 184 ++++++++++++++++++++++++++ holesky.tf | 34 +++++ 4 files changed, 349 insertions(+) create mode 100644 ansible/group_vars/nimbus.holesky.yml create mode 100644 ansible/holesky.yml create mode 100644 ansible/vars/layout/holesky.yml create mode 100644 holesky.tf diff --git a/ansible/group_vars/nimbus.holesky.yml b/ansible/group_vars/nimbus.holesky.yml new file mode 100644 index 0000000..0599e52 --- /dev/null +++ b/ansible/group_vars/nimbus.holesky.yml @@ -0,0 +1,81 @@ +--- +# Go-Ethereum ------------------------------------------------------------------ + +geth_service_name: 'geth-{{ geth_network_name }}-{{ "%02d"|format(idx|int+1) }}' +geth_network_name: 'holesky' +geth_sync_mode: 'full' +geth_log_level_name: 'info' +# Geth auth & JWT token +geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}' +geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}' +# Memory settings +geth_cont_mem_ratio: 0.2 +# Ports +geth_rpc_enabled: true +geth_websocket_enabled: true +geth_rpc_addr: '0.0.0.0' +geth_authrpc_addr: '0.0.0.0' +geth_websocket_addr: '0.0.0.0' +geth_port: '{{ 30303 + idx|int + 11 }}' +geth_metrics_port: '{{ 6060 + idx|int + 11 }}' +geth_rpc_port: '{{ 8545 + idx|int + 11 }}' +geth_authrpc_port: '{{ 8551 + idx|int + 11 }}' +geth_websocket_port: '{{ 9546 + idx|int + 11 }}' + +# Beacon Nodes ----------------------------------------------------------------- + +beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ beacon_node_repo_branch }}-{{ "%02d"|format(idx|int+1) }}' +beacon_node_network: 'holesky' +beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}' +# We map short names to branches to avoid too long service names. +node_name_to_branch_map: + libp2p: 'nim-libp2p-auto-bump-unstable' + chronos: 'nim-chronos-auto-bump-unstable' +# Builds +beacon_node_build_frequency: 'daily' +beacon_node_build_nim_flags: '-d:noSignalHandler {{ node.get("nim_flags", "") }}' +# Monitoring +beacon_node_validator_monitor_auto: true +beacon_node_validator_monitor_details: >- + {{ (node.public_api is not defined or not node.public_api) + and (node.end is defined and (node.end - node.start) <= 64) }} +# Execution layer Enginer API +beacon_node_exec_layer_urls: ['http://localhost:{{ geth_authrpc_port }}'] +beacon_node_exec_layer_jwt_secret: '{{ geth_authrpc_jwtsecret | mandatory }}' +# Suggests it to the Execution Layer client. +beacon_node_suggested_fee_recipient: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="address")}}' +# Validators from nimbus-private repo¬ +beacon_node_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}' +beacon_node_dist_validators_start: '{{ node.get("vc", false) | ternary(0, node.start) | mandatory }}' +beacon_node_dist_validators_end: '{{ node.get("vc", false) | ternary(0, node.end) | mandatory }}' +# Ports +beacon_node_discovery_port: '{{ 9000 + idx|int + 11 }}' +beacon_node_listening_port: '{{ 9000 + idx|int + 11 }}' +beacon_node_metrics_port: '{{ 9200 + idx|int + 11 }}' +beacon_node_rest_port: '{{ 9300 + idx|int + 11 }}' +beacon_node_rest_address: '0.0.0.0' + +# Validator Client ------------------------------------------------------------- + +validator_client_service_name: 'validator-client-{{ validator_client_network }}-{{ validator_client_build_repo_branch }}-{{ "%02d"|format(idx|int+1) }}' +validator_client_service_enabled: '{{ node.get("vc", false) }}' +validator_client_network: '{{ beacon_node_network }}' +validator_client_log_level: 'INFO' +validator_client_beacon_node_url: 'http://127.0.0.1:{{ beacon_node_rest_port }}' +# Builds +validator_client_build_repo_branch: '{{ beacon_node_repo_branch }}' +validator_client_build_frequency: 'daily' +# Ports +validator_client_metrics_port: '{{ 8108 + idx|int + 1 }}' +validator_client_keymanager_port: '{{ 5052 + idx|int + 1 }}' +# Suggests it to the Execution Layer client and the builder network. +validator_client_suggested_fee_recipient: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="address")}}' +# Keymanager +validator_client_keymanager_enabled: true +validator_client_keymanager_token: '{{lookup("bitwarden", "nimbus/keymanager", field="token")}}' +# Validators Distribution +validator_client_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}' +validator_client_dist_validators_start: '{{ (not node.get("vc", false)) | ternary(0, node.start) | mandatory }}' +validator_client_dist_validators_end: '{{ (not node.get("vc", false)) | ternary(0, node.end) | mandatory }}' + +# Fleet layout can be found in: ansible/vars/layout/holesky.yml diff --git a/ansible/holesky.yml b/ansible/holesky.yml new file mode 100644 index 0000000..4878149 --- /dev/null +++ b/ansible/holesky.yml @@ -0,0 +1,50 @@ +--- +- name: Verify Ansible versions + hosts: all + tags: always + become: false + run_once: true + gather_facts: false + tasks: + - local_action: command ./versioncheck.py + changed_when: false + +- name: Deploy prater Hetzner linux nodes + become: true + serial: '{{ serial|default(1) }}' + hosts: nimbus-prater-metal + roles: + - { role: swap-file, tags: [ swap-file ] } + - { role: open-ports, tags: [ open-ports ] } + - { role: smart-metrics, tags: [ smart-metrics ] } + - { role: redirect-ports, tags: [ redirect-ports ] } + - { role: nimbus-era-files, tags: [ nimbus-era-files ] } + tasks: + - include_role: name=infra-role-geth + when: node.el == "geth" + with_items: '{{ nodes_layout[hostname] }}' + loop_control: { loop_var: node, index_var: idx } + + - include_role: name=infra-role-geth-exporter + when: node.el == "geth" + with_items: '{{ nodes_layout[hostname] }}' + loop_control: { loop_var: node, index_var: idx } + + - include_role: name=infra-role-erigon + when: node.el == "erigon" + with_items: '{{ nodes_layout[hostname] }}' + loop_control: { loop_var: node, index_var: idx } + + - include_role: name=infra-role-nethermind + when: node.el == "nethermind" + with_items: '{{ nodes_layout[hostname] }}' + loop_control: { loop_var: node, index_var: idx } + + - include_role: name=infra-role-beacon-node-linux + with_items: '{{ nodes_layout[hostname] }}' + loop_control: { loop_var: node, index_var: idx } + + - include_role: name=infra-role-validator-client + with_items: '{{ nodes_layout[hostname] }}' + when: validator_client_service_enabled + loop_control: { loop_var: node, index_var: idx } diff --git a/ansible/vars/layout/holesky.yml b/ansible/vars/layout/holesky.yml new file mode 100644 index 0000000..2f25480 --- /dev/null +++ b/ansible/vars/layout/holesky.yml @@ -0,0 +1,184 @@ +--- +nodes_layout: + # Geth ------------------------------------------------------------------------- + 'linux-01.ih-eu-mda1.nimbus.holesky': # 0 each + - { branch: 'stable', el: 'geth', vc: true } + - { branch: 'testing', el: 'geth', vc: false } + - { branch: 'unstable', el: 'geth', vc: false } + - { branch: 'libp2p', el: 'geth', vc: false } + + 'linux-02.ih-eu-mda1.nimbus.holesky': # 1 each + - { branch: 'stable', start: 0, end: 1, el: 'geth', vc: true } + - { branch: 'testing', start: 1, end: 2, el: 'geth', vc: false } + - { branch: 'unstable', start: 2, end: 3, el: 'geth', vc: false } + - { branch: 'libp2p', start: 3, end: 4, el: 'geth', vc: false } + + 'linux-03.ih-eu-mda1.nimbus.holesky': # 5 each + - { branch: 'stable', start: 4, end: 9, el: 'geth', vc: true } + - { branch: 'testing', start: 9, end: 14, el: 'geth', vc: false } + - { branch: 'unstable', start: 14, end: 19, el: 'geth', vc: false } + - { branch: 'libp2p', start: 19, end: 24, el: 'geth', vc: false } + + 'linux-04.ih-eu-mda1.nimbus.holesky': # 14 each + - { branch: 'stable', start: 24, end: 38, el: 'geth', vc: true } + - { branch: 'testing', start: 38, end: 52, el: 'geth', vc: false } + - { branch: 'unstable', start: 52, end: 66, el: 'geth', vc: false } + - { branch: 'libp2p', start: 66, end: 80, el: 'geth', vc: false } + + 'linux-05.ih-eu-mda1.nimbus.holesky': # 20 each + - { branch: 'stable', start: 80, end: 100, el: 'geth', vc: true } + - { branch: 'testing', start: 100, end: 120, el: 'geth', vc: false } + - { branch: 'unstable', start: 120, end: 140, el: 'geth', vc: false } + - { branch: 'libp2p', start: 140, end: 160, el: 'geth', vc: false } + + 'linux-06.ih-eu-mda1.nimbus.holesky': # 110 each + - { branch: 'stable', start: 160, end: 270, el: 'geth', vc: true } + - { branch: 'testing', start: 270, end: 380, el: 'geth', vc: false } + - { branch: 'unstable', start: 380, end: 490, el: 'geth', vc: false } + - { branch: 'libp2p', start: 490, end: 600, el: 'geth', vc: false } + + 'linux-07.ih-eu-mda1.nimbus.holesky': # 400 each + - { branch: 'stable', start: 600, end: 1000, el: 'geth', vc: true } + - { branch: 'testing', start: 1000, end: 1400, el: 'geth', vc: false } + - { branch: 'unstable', start: 1400, end: 1800, el: 'geth', vc: false } + - { branch: 'libp2p', start: 1800, end: 2200, el: 'geth', vc: false } + + 'linux-08.ih-eu-mda1.nimbus.holesky': # 700 each + - { branch: 'stable', start: 2200, end: 2900, el: 'geth', vc: true } + - { branch: 'testing', start: 2900, end: 3600, el: 'geth', vc: false } + - { branch: 'unstable', start: 3600, end: 4300, el: 'geth', vc: false } + - { branch: 'libp2p', start: 4300, end: 5000, el: 'geth', vc: false } + + 'linux-09.ih-eu-mda1.nimbus.holesky': # 2000 each + - { branch: 'stable', start: 5000, end: 7000, el: 'geth', vc: true } + - { branch: 'testing', start: 7000, end: 9000, el: 'geth', vc: false } + - { branch: 'unstable', start: 9000, end: 11000, el: 'geth', vc: false } + - { branch: 'libp2p', start: 11000, end: 13000, el: 'geth', vc: false } + + 'linux-10.ih-eu-mda1.nimbus.holesky': # 5000 each + - { branch: 'stable', start: 13000, end: 18000, el: 'geth', vc: true } + - { branch: 'testing', start: 18000, end: 23000, el: 'geth', vc: false } + - { branch: 'unstable', start: 23000, end: 28000, el: 'geth', vc: false } + - { branch: 'libp2p', start: 28000, end: 33000, el: 'geth', vc: false } + + # Erigon ------------------------------------------------------------------------ + 'linux-11.ih-eu-mda1.nimbus.holesky': # 0 each + - { branch: 'stable', el: 'erigon', vc: false } + - { branch: 'testing', el: 'erigon', vc: true } + - { branch: 'unstable', el: 'erigon', vc: false } + - { branch: 'libp2p', el: 'erigon', vc: false } + + 'linux-12.ih-eu-mda1.nimbus.holesky': # 1 each + - { branch: 'stable', start: 33000, end: 33001, el: 'erigon', vc: false } + - { branch: 'testing', start: 33001, end: 33002, el: 'erigon', vc: true } + - { branch: 'unstable', start: 33002, end: 33003, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 33003, end: 33004, el: 'erigon', vc: false } + + 'linux-13.ih-eu-mda1.nimbus.holesky': # 5 each + - { branch: 'stable', start: 33004, end: 33009, el: 'erigon', vc: false } + - { branch: 'testing', start: 33009, end: 33014, el: 'erigon', vc: true } + - { branch: 'unstable', start: 33014, end: 33019, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 33019, end: 33024, el: 'erigon', vc: false } + + 'linux-14.ih-eu-mda1.nimbus.holesky': # 14 each + - { branch: 'stable', start: 33024, end: 33038, el: 'erigon', vc: false } + - { branch: 'testing', start: 33038, end: 33052, el: 'erigon', vc: true } + - { branch: 'unstable', start: 33052, end: 33066, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 33066, end: 33080, el: 'erigon', vc: false } + + 'linux-15.ih-eu-mda1.nimbus.holesky': # 20 each + - { branch: 'stable', start: 33080, end: 33100, el: 'erigon', vc: false } + - { branch: 'testing', start: 33100, end: 33120, el: 'erigon', vc: true } + - { branch: 'unstable', start: 33120, end: 33140, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 33140, end: 33160, el: 'erigon', vc: false } + + 'linux-16.ih-eu-mda1.nimbus.holesky': # 110 each + - { branch: 'stable', start: 33160, end: 33270, el: 'erigon', vc: false } + - { branch: 'testing', start: 33270, end: 33380, el: 'erigon', vc: true } + - { branch: 'unstable', start: 33380, end: 33490, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 33490, end: 33600, el: 'erigon', vc: false } + + 'linux-17.ih-eu-mda1.nimbus.holesky': # 400 each + - { branch: 'stable', start: 33600, end: 34000, el: 'erigon', vc: false } + - { branch: 'testing', start: 34000, end: 34400, el: 'erigon', vc: true } + - { branch: 'unstable', start: 34400, end: 34800, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 34800, end: 35200, el: 'erigon', vc: false } + + 'linux-18.ih-eu-mda1.nimbus.holesky': # 700 each + - { branch: 'stable', start: 35200, end: 35900, el: 'erigon', vc: false } + - { branch: 'testing', start: 35900, end: 36600, el: 'erigon', vc: true } + - { branch: 'unstable', start: 36600, end: 37300, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 37300, end: 38000, el: 'erigon', vc: false } + + 'linux-19.ih-eu-mda1.nimbus.holesky': # 2000 each + - { branch: 'stable', start: 38000, end: 40000, el: 'erigon', vc: false } + - { branch: 'testing', start: 40000, end: 42000, el: 'erigon', vc: true } + - { branch: 'unstable', start: 42000, end: 44000, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 44000, end: 46000, el: 'erigon', vc: false } + + 'linux-20.ih-eu-mda1.nimbus.holesky': # 5000 each + - { branch: 'stable', start: 46000, end: 51000, el: 'erigon', vc: false } + - { branch: 'testing', start: 51000, end: 56000, el: 'erigon', vc: true } + - { branch: 'unstable', start: 56000, end: 61000, el: 'erigon', vc: false } + - { branch: 'libp2p', start: 61000, end: 66000, el: 'erigon', vc: false } + + # Nethermind --------------------------------------------------------------------- + 'linux-21.ih-eu-mda1.nimbus.holesky': # 0 each + - { branch: 'stable', el: 'nethermind', vc: false } + - { branch: 'testing', el: 'nethermind', vc: false } + - { branch: 'unstable', el: 'nethermind', vc: true } + - { branch: 'libp2p', el: 'nethermind', vc: false } + + 'linux-22.ih-eu-mda1.nimbus.holesky': # 1 each + - { branch: 'stable', start: 66000, end: 66001, el: 'nethermind', vc: false } + - { branch: 'testing', start: 66001, end: 66002, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 66002, end: 66003, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 66003, end: 66004, el: 'nethermind', vc: false } + + 'linux-23.ih-eu-mda1.nimbus.holesky': # 5 each + - { branch: 'stable', start: 66004, end: 66009, el: 'nethermind', vc: false } + - { branch: 'testing', start: 66009, end: 66014, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 66014, end: 66019, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 66019, end: 66024, el: 'nethermind', vc: false } + + 'linux-24.ih-eu-mda1.nimbus.holesky': # 14 each + - { branch: 'stable', start: 66024, end: 66038, el: 'nethermind', vc: false } + - { branch: 'testing', start: 66038, end: 66052, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 66052, end: 66066, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 66066, end: 66080, el: 'nethermind', vc: false } + + 'linux-25.ih-eu-mda1.nimbus.holesky': # 20 each + - { branch: 'stable', start: 66080, end: 66100, el: 'nethermind', vc: false } + - { branch: 'testing', start: 66100, end: 66120, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 66120, end: 66140, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 66140, end: 66160, el: 'nethermind', vc: false } + + 'linux-26.ih-eu-mda1.nimbus.holesky': # 110 each + - { branch: 'stable', start: 66160, end: 66270, el: 'nethermind', vc: false } + - { branch: 'testing', start: 66270, end: 66380, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 66380, end: 66490, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 66490, end: 66600, el: 'nethermind', vc: false } + + 'linux-27.ih-eu-mda1.nimbus.holesky': # 400 each + - { branch: 'stable', start: 66600, end: 67000, el: 'nethermind', vc: false } + - { branch: 'testing', start: 67000, end: 67400, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 67400, end: 67800, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 67800, end: 68200, el: 'nethermind', vc: false } + + 'linux-28.ih-eu-mda1.nimbus.holesky': # 700 each + - { branch: 'stable', start: 68200, end: 68900, el: 'nethermind', vc: false } + - { branch: 'testing', start: 68900, end: 69600, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 69600, end: 70300, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 70300, end: 71000, el: 'nethermind', vc: false } + + 'linux-29.ih-eu-mda1.nimbus.holesky': # 2000 each + - { branch: 'stable', start: 71000, end: 73000, el: 'nethermind', vc: false } + - { branch: 'testing', start: 73000, end: 75000, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 75000, end: 77000, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 77000, end: 79000, el: 'nethermind', vc: false } + + 'linux-30.ih-eu-mda1.nimbus.holesky': # 5000 each + - { branch: 'stable', start: 79000, end: 84000, el: 'nethermind', vc: false } + - { branch: 'testing', start: 84000, end: 89000, el: 'nethermind', vc: false } + - { branch: 'unstable', start: 89000, end: 94000, el: 'nethermind', vc: true } + - { branch: 'libp2p', start: 94000, end:100000, el: 'nethermind', vc: false } diff --git a/holesky.tf b/holesky.tf new file mode 100644 index 0000000..16f0883 --- /dev/null +++ b/holesky.tf @@ -0,0 +1,34 @@ +module "nimbus_nodes_holesky_innova" { + source = "github.com/status-im/infra-tf-dummy-module" + + name = "linux" + env = "nimbus" + stage = "holesky" + group = "nimbus-holesky-linux" + region = "eu-mda1" + prefix = "ih" + domain = var.domain + + ips = [ + "12.34.56.78", # linux-01.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-02.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-03.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-04.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-05.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-06.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-07.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-08.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-09.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-10.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-11.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-12.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-13.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-14.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-15.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-16.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-17.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-18.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-19.ih-eu-mda1.nimbus.holesky + "12.34.56.78", # linux-20.ih-eu-mda1.nimbus.holesky + ] +}