Anton Iakimov 2024-10-08 15:57:57 +02:00
parent 42774ce4a6
commit e6518f0a02
No known key found for this signature in database
21 changed files with 301 additions and 155 deletions

View File

@ -8,8 +8,8 @@ These are [Beacon API](https://ethereum.github.io/beacon-APIs/) endpoints intend
| Endpoint | Host |
|-------------------------------------------------|--------------------------------------|
| http://unstable.mainnet.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.mainnet` |
| http://testing.mainnet.beacon-api.nimbus.team/ | `linux-02.ih-eu-mda1.nimbus.mainnet` |
| http://unstable.mainnet.beacon-api.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.mainnet` |
| http://testing.mainnet.beacon-api.nimbus.team/ | `geth-02.ih-eu-mda1.nimbus.mainnet` |
| http://unstable.sepolia.beacon-api.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
| http://unstable.holesky.beacon-api.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.holesky` |
| http://testing.holesky.beacon-api.nimbus.team/ | `geth-02.ih-eu-mda1.nimbus.holesky` |
@ -18,12 +18,12 @@ These nodes have no validators attached.
There are also archives of ERA files:
| Endpoint | Host |
|-----------------------------------|--------------------------------------|
| https://mainnet.era.nimbus.team/ | `linux-03.ih-eu-mda1.nimbus.mainnet` |
| https://sepolia.era.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
| https://sepolia.era1.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
| https://holesky.era.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.holesky` |
| Endpoint | Host |
|-----------------------------------|---------------------------------------|
| https://mainnet.era.nimbus.team/ | `erigon-01.ih-eu-mda1.nimbus.mainnet` |
| https://sepolia.era.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
| https://sepolia.era1.nimbus.team/ | `linux-01.ih-eu-mda1.nimbus.sepolia` |
| https://holesky.era.nimbus.team/ | `geth-01.ih-eu-mda1.nimbus.holesky` |
# Dashboards

View File

@ -12,7 +12,7 @@
- name: Configure ERA files hosting
become: true
hosts:
- linux-03.ih-eu-mda1.nimbus.mainnet
- erigon-01.ih-eu-mda1.nimbus.mainnet
- linux-01.ih-eu-mda1.nimbus.sepolia
- geth-01.ih-eu-mda1.nimbus.holesky
roles:

View File

@ -6,82 +6,128 @@ bootstrap__extra_volume_bind_path: null
# SWAP
swap_file_size_mb: 2048
# Go-Ethereum
geth_service_name: 'geth-{{ geth_network_name }}'
# Ports used for all 3 EL nodes as they are mutually exclusive.
exec_layer_p2p_port: '{{ 30303 + 3*idx|int + 1 }}'
exec_layer_rpc_port: '{{ 8545 + idx|int + 1 }}'
exec_layer_authrpc_port: '{{ 8551 + idx|int + 1 }}'
exec_layer_metrics_port: '{{ 6060 + idx|int + 1 }}'
# Go-Ethereum ------------------------------------------------------------------
geth_service_name: 'geth-{{ geth_network_name }}-{{ node.branch | mandatory }}'
geth_service_path: '/docker/{{ geth_service_name }}'
geth_network_name: 'mainnet'
geth_cont_name: '{{ geth_service_name }}-node'
geth_cont_vol: '{{ geth_service_path }}/node'
geth_sync_mode: 'snap'
geth_log_level_name: 'info'
# Geth auth & JWT token
geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
# Memory settings¬
geth_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
# Memory settings
geth_cont_mem_ratio: 0.15
geth_cache_size: '{{ (ansible_memtotal_mb * 0.05|float) | int }}'
# Ports
geth_port: 30303
geth_rpc_addr: '0.0.0.0'
geth_rpc_port: 8545
geth_authrpc_addr: '127.0.0.1'
geth_authrpc_port: 8551
# Geth metrics¬
geth_rpc_addr: '0.0.0.0'
geth_authrpc_addr: '127.0.0.1'
geth_port: '{{ exec_layer_p2p_port }}'
geth_rpc_port: '{{ exec_layer_rpc_port }}'
geth_authrpc_port: '{{ exec_layer_authrpc_port }}'
geth_metrics_port: '{{ exec_layer_metrics_port }}'
# Geth metrics
geth_expo_service_name: '{{ geth_service_name }}'
geth_expo_source_cont_name: '{{ geth_cont_name }}'
geth_expo_source_data_path: '{{ geth_cont_vol }}/data'
geth_expo_cont_port: 9400
geth_expo_cont_port: '{{ 9400 + (idx|int) + 1 }}'
# Erigon -----------------------------------------------------------------------
erigon_network_name: 'holesky'
erigon_service_name: 'erigon-{{ geth_network_name }}-{{ node.branch | mandatory }}'
erigon_prune: 'hrtc' # TODO: Possibly wrong, verify.
erigon_log_level: 'info'
erigon_cont_mem_ratio: 0.15
erigon_max_peers: 20
erigon_miner_enabled: true
# Ports
erigon_metrics_enabled: true
erigon_rpc_addr: '0.0.0.0'
erigon_authrpc_addr: '0.0.0.0'
erigon_port: '{{ exec_layer_p2p_port }}'
erigon_p2p_allowed_ports: ['{{ erigon_port }}', '{{ erigon_port|int + 1 }}']
erigon_rpc_port: '{{ exec_layer_rpc_port }}'
erigon_metrics_port: '{{ exec_layer_metrics_port }}'
erigon_authrpc_port: '{{ exec_layer_authrpc_port }}'
erigon_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
# Nimbus ETH1 ------------------------------------------------------------------
nimbus_eth1_service_name: 'nel-{{ geth_network_name }}-{{ node.branch | mandatory }}'
nimbus_eth1_repo_branch: 'master'
nimbus_eth1_network: 'mainnet'
nimbus_eth1_max_peers: 160
nimbus_eth1_log_level: 'DEBUG'
nimbus_eth1_service_path: '/docker/{{ nimbus_eth1_service_name }}'
nimbus_eth1_engine_http_enabled: true
nimbus_eth1_engine_ws_enabled: true
nimbus_eth1_era_dir: '{{ nimbus_era_files_timer_path }}'
nimbus_eth1_era1_dir: '/docker/era1'
# Ports
nimbus_eth1_listening_port: '{{ exec_layer_p2p_port }}'
nimbus_eth1_discovery_port: '{{ exec_layer_p2p_port }}'
nimbus_eth1_http_port: '{{ exec_layer_rpc_port }}'
nimbus_eth1_metrics_port: '{{ exec_layer_metrics_port }}'
nimbus_eth1_metrics_address: '0.0.0.0'
# API secert
nimbus_eth1_jwt_secret: '{{ beacon_node_exec_layer_jwt_secret }}'
# Beacon Nodes -----------------------------------------------------------------
# Nimbus Beacon Node
beacon_node_network: 'mainnet'
beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}'
beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}'
# Tuning
beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
# We map short names to branches to avoid too long service names.
node_name_to_branch_map:
libp2p: 'nim-libp2p-auto-bump-unstable'
# TODO Drop this once all nodes have an number.
service_number: '{{ node.get("num", False) | ternary(("-%02d"|format(node.get("num", 0))), "") }}'
beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}{{ service_number }}'
beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
# Ports
beacon_node_discovery_port: '{{ 9000 + idx }}'
beacon_node_listening_port: '{{ 9000 + idx }}'
beacon_node_metrics_port: '{{ 9200 + idx }}'
beacon_node_rest_port: '{{ beacon_node_rest_port_base + idx }}'
beacon_node_rest_port_base: 9300
beacon_node_rest_address: '0.0.0.0'
beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
# Firewall
beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
# Builds
beacon_node_update_build_targets: ['nimbus_beacon_node', 'ncli_db']
beacon_node_update_build: '{{ beacon_node_repo_branch != "stable" }}'
beacon_node_update_frequency: '*-*-* {{ "%02d" | format(idx * 2) }}:00:00'
# Tuning
beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
# FIXME: Temporary test to debug REST API timeout issues.
# https://github.com/status-im/nimbus-eth2/issues/5838
#beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'
beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
# Monitoring
beacon_node_validator_monitor_auto: true
beacon_node_validator_monitor_details: '{{ node.public_api is not defined or not node.public_api }}'
# Mainnet validators run on a separate fleet.
beacon_node_dist_validators_enabled: false
# Excellent stress test and good service to the community.
beacon_node_subscribe_all: true
beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
# Execution layer Enginer API
beacon_node_exec_layer_jwt_secret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
beacon_node_exec_layer_urls_local: ['http://localhost:{{ exec_layer_authrpc_port }}']
beacon_node_exec_layer_urls: '{{ beacon_node_exec_layer_urls_local if node.get("el") else [] }}'
# Light client data
beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
beacon_node_light_client_data_serve: true
beacon_node_light_client_data_import_mode: 'full'
# Execution layer Enginer API
beacon_node_exec_layer_jwt_secret: '{{ geth_authrpc_jwtsecret }}'
beacon_node_exec_layer_urls_local_geth: ['http://localhost:{{ geth_authrpc_port|string }}']
beacon_node_exec_layer_urls_all: '{{ beacon_node_exec_layer_urls_discovered_geth + beacon_node_exec_layer_urls_local_geth }}'
beacon_node_exec_layer_urls: |-
{{ beacon_node_exec_layer_urls_all if inventory_hostname_short == "linux-01"
else (idx % 2 == 0) | ternary(beacon_node_exec_layer_urls_all, []) }}
# Mainnet validators run on a separate fleet.
beacon_node_dist_validators_enabled: false
# Ports
beacon_node_rest_port_base: 9300
beacon_node_rest_port: '{{ beacon_node_rest_port_base + idx|int + 1 }}'
beacon_node_discovery_port: '{{ 9000 + idx|int + 1 }}'
beacon_node_listening_port: '{{ 9000 + idx|int + 1 }}'
beacon_node_metrics_port: '{{ 9200 + idx|int + 1 }}'
beacon_node_rest_address: '0.0.0.0'
beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
# Firewall
beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
# Excellent stress test and good service to the community.
beacon_node_subscribe_all: true
# FIXME: Temporary test to debug REST API timeout issues.
# https://github.com/status-im/nimbus-eth2/issues/5838
#beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'
# Reduce Consul alerts sensitivity
beacon_node_consul_check_disabled: '{{ node.get("public_api", false) }}'
@ -105,29 +151,31 @@ nimbus_era_files_timer_enabled: '{{ (nodes_layout[inventory_hostname]|length) >
nimbus_era_files_timer_path: '/data/era'
nimbus_era_files_network: '{{ beacon_node_network }}'
# FIXME: Not pretty, since hardcoded, but the simplest way to do it right now.
nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-stable-01/data/db'
nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-stable-01/repo/build/ncli_db'
nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-stable/data/db'
nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-stable/repo/build/ncli_db'
# Migrated to NFTables from IPTables.
# https://github.com/status-im/infra-misc/issues/301
bootstrap__firewall_nftables: true
# Open Ports
host_el_type: '{{ ansible_hostname|split("-")|first }}'
open_ports_list:
nginx:
- { port: '443', comment: 'Nginx' }
smart-metrics:
- { port: '{{ smart_metrics_listen_port }}', comment: 'SMART Metrics', ipset: 'hq.metrics', iifname: 'wg0' }
geth-node:
- { port: '{{ geth_port }}', comment: 'Go-Ethereum libp2p', protocol: 'tcp' }
- { port: '{{ geth_port }}', comment: 'Go-Ethereum discovery', protocol: 'udp' }
- { port: '{{ geth_expo_cont_port }}', comment: 'Go-Ethereum Exporter', ipset: 'hq.metrics', iifname: 'wg0' }
- { port: '{{ geth_authrpc_port }}', comment: 'Go-Ethereum AuthRPC', ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
- { port: '{{ geth_rpc_port }}', comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy', iifname: 'wg0' }
exec-node:
- { port: '30304-30315', comment: '{{ host_el_type }} LibP2P', protocol: 'tcp' }
- { port: '30304-30315', comment: '{{ host_el_type }} Discovery', protocol: 'udp' }
- { port: '6061-6064', comment: '{{ host_el_type }} Metrics', ipset: 'hq.metrics', iifname: 'wg0' }
- { port: '9401-9404', comment: '{{ host_el_type }} Exporter', ipset: 'hq.metrics', iifname: 'wg0' }
- { port: '8552-8555', comment: 'Go-Ethereum AuthRPC', ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
- { port: '8446-8449', comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy', iifname: 'wg0' }
beacon-node:
- { port: '9000-9005', comment: 'Beacon Node libp2p', protocol: 'tcp' }
- { port: '9000-9005', comment: 'Beacon Node discovery', protocol: 'udp' }
- { port: '9200-9205', comment: 'Beacon Node Metrics', ipset: 'hq.metrics', iifname: 'wg0' }
- { port: '9300-9305', comment: 'Beacon Node REST API', ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
- { port: '9001-9004', comment: 'Beacon Node libp2p', protocol: 'tcp' }
- { port: '9001-9004', comment: 'Beacon Node discovery', protocol: 'udp' }
- { port: '9201-9204', comment: 'Beacon Node Metrics', ipset: 'hq.metrics', iifname: 'wg0' }
- { port: '9301-9304', comment: 'Beacon Node REST API', ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
# Fleet layout can be found in: ansible/vars/layout/mainnet.yml

View File

@ -15,4 +15,4 @@ redirect_ports:
public-beacon-api:
src: 80
dst: '{{ test_api_beacon_node_rest_port }}'
comment: 'Public Beacon API ({{ test_api_beacon_node_branch }}-0{{ test_api_beacon_node_rest_index|int - 1 }})'
comment: 'Public Beacon API ({{ test_api_beacon_node_branch }})'

View File

@ -16,4 +16,4 @@ redirect_ports:
public-beacon-api:
src: 80
dst: '{{ test_api_beacon_node_rest_port }}'
comment: 'Public Beacon API ({{ test_api_beacon_node_branch }}-0{{ test_api_beacon_node_rest_index|int - 1 }})'
comment: 'Public Beacon API ({{ test_api_beacon_node_branch }})'

View File

@ -6,7 +6,9 @@ bench-02.he-eu-hel1.nimbus.eth1 ansible_host=65.21.46.216 data_center=he-eu-hel1
bootstrap-01.aws-eu-central-1a.nimbus.mainnet ansible_host=3.120.104.18 data_center=aws-eu-central-1a dns_domain=status.im dns_entry=bootstrap-01.aws-eu-central-1a.nimbus.mainnet.status.im env=nimbus region=eu-central-1a stage=mainnet
bootstrap-02.aws-eu-central-1a.nimbus.mainnet ansible_host=3.64.117.223 data_center=aws-eu-central-1a dns_domain=status.im dns_entry=bootstrap-02.aws-eu-central-1a.nimbus.mainnet.status.im env=nimbus region=eu-central-1a stage=mainnet
erigon-01.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.148 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-01.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
erigon-01.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.99 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-01.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
erigon-02.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.149 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-02.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
erigon-02.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.100 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-02.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
erigon-03.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.151 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-03.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
erigon-04.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.157 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-04.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
erigon-05.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.241 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-05.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
@ -16,7 +18,9 @@ erigon-08.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.244 data_center=ih-eu
erigon-09.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.245 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-09.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
erigon-10.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.246 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=erigon-10.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
geth-01.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.71 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-01.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
geth-01.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.101 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-01.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
geth-02.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.81 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-02.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
geth-02.ih-eu-mda1.nimbus.mainnet ansible_host=185.181.229.82 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-02.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
geth-03.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.82 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-03.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
geth-04.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.107 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-04.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
geth-05.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.129 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-05.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
@ -27,19 +31,15 @@ geth-09.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.141 data_center=ih-eu-m
geth-10.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.147 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=geth-10.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
holesky-01.ih-eu-mda1.nimbus.geth ansible_host=194.33.40.232 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=holesky-01.ih-eu-mda1.nimbus.geth.status.im env=nimbus region=eu-mda1 stage=geth
holesky-02.ih-eu-mda1.nimbus.geth ansible_host=194.33.40.233 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=holesky-02.ih-eu-mda1.nimbus.geth.status.im env=nimbus region=eu-mda1 stage=geth
linux-01.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.70 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-01.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-01.ih-eu-mda1.nimbus.sepolia ansible_host=185.181.230.77 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-01.ih-eu-mda1.nimbus.sepolia.status.im env=nimbus region=eu-mda1 stage=sepolia
linux-02.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.72 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-02.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-03.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.78 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-03.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-04.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.99 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-04.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-05.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.100 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-05.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-06.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.101 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-06.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
linux-07.ih-eu-mda1.nimbus.mainnet ansible_host=185.181.229.82 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=linux-07.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
macm2-01.ih-eu-mda1.nimbus.holesky ansible_host=185.181.230.76 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=macm2-01.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
mainnet-01.aws-eu-central-1a.nimbus.geth ansible_host=18.196.208.138 data_center=aws-eu-central-1a dns_domain=status.im dns_entry=mainnet-01.aws-eu-central-1a.nimbus.geth.status.im env=nimbus region=eu-central-1a stage=geth
metal-01.ih-eu-mda1.nimbus.eth1 ansible_host=194.33.40.237 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=metal-01.ih-eu-mda1.nimbus.eth1.status.im env=nimbus region=eu-mda1 stage=eth1
metal-01.ih-eu-mda1.nimbus.fluffy ansible_host=194.33.40.238 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=metal-01.ih-eu-mda1.nimbus.fluffy.status.im env=nimbus region=eu-mda1 stage=fluffy
metal-02.ih-eu-mda1.nimbus.fluffy ansible_host=194.33.40.239 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=metal-02.ih-eu-mda1.nimbus.fluffy.status.im env=nimbus region=eu-mda1 stage=fluffy
nel-01.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.70 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=nel-01.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
nel-02.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.72 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=nel-02.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
nel-03.ih-eu-mda1.nimbus.mainnet ansible_host=194.33.40.78 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=nel-03.ih-eu-mda1.nimbus.mainnet.status.im env=nimbus region=eu-mda1 stage=mainnet
neth-01.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.247 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=neth-01.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
neth-02.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.248 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=neth-02.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
neth-03.ih-eu-mda1.nimbus.holesky ansible_host=194.33.40.249 data_center=ih-eu-mda1 dns_domain=status.im dns_entry=neth-03.ih-eu-mda1.nimbus.holesky.status.im env=nimbus region=eu-mda1 stage=holesky
@ -74,7 +74,9 @@ store-03.he-eu-hel1.logs.nimbus
[ih-eu-mda1]
erigon-01.ih-eu-mda1.nimbus.holesky
erigon-01.ih-eu-mda1.nimbus.mainnet
erigon-02.ih-eu-mda1.nimbus.holesky
erigon-02.ih-eu-mda1.nimbus.mainnet
erigon-03.ih-eu-mda1.nimbus.holesky
erigon-04.ih-eu-mda1.nimbus.holesky
erigon-05.ih-eu-mda1.nimbus.holesky
@ -84,7 +86,9 @@ erigon-08.ih-eu-mda1.nimbus.holesky
erigon-09.ih-eu-mda1.nimbus.holesky
erigon-10.ih-eu-mda1.nimbus.holesky
geth-01.ih-eu-mda1.nimbus.holesky
geth-01.ih-eu-mda1.nimbus.mainnet
geth-02.ih-eu-mda1.nimbus.holesky
geth-02.ih-eu-mda1.nimbus.mainnet
geth-03.ih-eu-mda1.nimbus.holesky
geth-04.ih-eu-mda1.nimbus.holesky
geth-05.ih-eu-mda1.nimbus.holesky
@ -95,18 +99,14 @@ geth-09.ih-eu-mda1.nimbus.holesky
geth-10.ih-eu-mda1.nimbus.holesky
holesky-01.ih-eu-mda1.nimbus.geth
holesky-02.ih-eu-mda1.nimbus.geth
linux-01.ih-eu-mda1.nimbus.mainnet
linux-01.ih-eu-mda1.nimbus.sepolia
linux-02.ih-eu-mda1.nimbus.mainnet
linux-03.ih-eu-mda1.nimbus.mainnet
linux-04.ih-eu-mda1.nimbus.mainnet
linux-05.ih-eu-mda1.nimbus.mainnet
linux-06.ih-eu-mda1.nimbus.mainnet
linux-07.ih-eu-mda1.nimbus.mainnet
macm2-01.ih-eu-mda1.nimbus.holesky
metal-01.ih-eu-mda1.nimbus.eth1
metal-01.ih-eu-mda1.nimbus.fluffy
metal-02.ih-eu-mda1.nimbus.fluffy
nel-01.ih-eu-mda1.nimbus.mainnet
nel-02.ih-eu-mda1.nimbus.mainnet
nel-03.ih-eu-mda1.nimbus.mainnet
neth-01.ih-eu-mda1.nimbus.holesky
neth-02.ih-eu-mda1.nimbus.holesky
neth-03.ih-eu-mda1.nimbus.holesky
@ -187,14 +187,18 @@ neth-10.ih-eu-mda1.nimbus.holesky
[nimbus-holesky-windows]
windows-01.ih-eu-mda1.nimbus.holesky
[nimbus-mainnet-metal]
linux-01.ih-eu-mda1.nimbus.mainnet
linux-02.ih-eu-mda1.nimbus.mainnet
linux-03.ih-eu-mda1.nimbus.mainnet
linux-04.ih-eu-mda1.nimbus.mainnet
linux-05.ih-eu-mda1.nimbus.mainnet
linux-06.ih-eu-mda1.nimbus.mainnet
linux-07.ih-eu-mda1.nimbus.mainnet
[nimbus-mainnet-erigon]
erigon-01.ih-eu-mda1.nimbus.mainnet
erigon-02.ih-eu-mda1.nimbus.mainnet
[nimbus-mainnet-geth]
geth-01.ih-eu-mda1.nimbus.mainnet
geth-02.ih-eu-mda1.nimbus.mainnet
[nimbus-mainnet-nel]
nel-01.ih-eu-mda1.nimbus.mainnet
nel-02.ih-eu-mda1.nimbus.mainnet
nel-03.ih-eu-mda1.nimbus.mainnet
[nimbus-mainnet-small]
bootstrap-01.aws-eu-central-1a.nimbus.mainnet
@ -254,13 +258,13 @@ windows-01.ih-eu-mda1.nimbus.holesky
[nimbus.mainnet]
bootstrap-01.aws-eu-central-1a.nimbus.mainnet
bootstrap-02.aws-eu-central-1a.nimbus.mainnet
linux-01.ih-eu-mda1.nimbus.mainnet
linux-02.ih-eu-mda1.nimbus.mainnet
linux-03.ih-eu-mda1.nimbus.mainnet
linux-04.ih-eu-mda1.nimbus.mainnet
linux-05.ih-eu-mda1.nimbus.mainnet
linux-06.ih-eu-mda1.nimbus.mainnet
linux-07.ih-eu-mda1.nimbus.mainnet
erigon-01.ih-eu-mda1.nimbus.mainnet
erigon-02.ih-eu-mda1.nimbus.mainnet
geth-01.ih-eu-mda1.nimbus.mainnet
geth-02.ih-eu-mda1.nimbus.mainnet
nel-01.ih-eu-mda1.nimbus.mainnet
nel-02.ih-eu-mda1.nimbus.mainnet
nel-03.ih-eu-mda1.nimbus.mainnet
[nimbus.sepolia]
linux-01.ih-eu-mda1.nimbus.sepolia

View File

@ -19,7 +19,7 @@
beacon_node_netkey: '{{lookup("bitwarden", "nimbus/netkey", field=hostname)}}'
roles:
- { role: infra-role-swap-file, tags: [ swap-file ] }
- { role: infra-role-open-ports, tags: [ open-ports ] }
- { role: infra-role-open-ports, tags: [ open-ports ] }
- { role: redirect-ports, tags: [ redirect-ports ] }
- { role: get-geth-api-urls, tags: [ get-geth-api-urls, beacon-node ] }
tasks:
@ -35,7 +35,10 @@
- name: Deploy mainnet linux beacon nodes
become: true
serial: '{{ serial|default(1) }}'
hosts: nimbus-mainnet-metal
hosts:
- nimbus-mainnet-geth
- nimbus-mainnet-erigon
- nimbus-mainnet-nel
vars_files: layout/mainnet.yml
vars:
# FIXME: Use all Geth nodes available in the fleet until we resync.
@ -46,8 +49,6 @@
- { role: infra-role-open-ports, tags: [ open-ports ] }
- { role: infra-role-smart-metrics, tags: [ smart-metrics ] }
- { role: redirect-ports, tags: [ redirect-ports ] }
- { role: infra-role-geth, tags: [ infra-role-geth ] }
- { role: infra-role-geth-exporter, tags: [ infra-role-geth-exporter ] }
- { role: get-geth-api-urls, tags: [ get-geth-api-urls, beacon-node ] }
- { role: nimbus-era-files, tags: [ nimbus-era-files ] }
tasks:
@ -59,3 +60,65 @@
loop_control:
loop_var: node
index_var: idx
- name: Deploy Mainnet Geth Nodes
become: true
serial: '{{ serial|default(1) }}'
hosts: nimbus-mainnet-geth
vars_files: layout/mainnet.yml
tasks:
- include_role:
name: infra-role-geth
apply:
tags: always
tags: [ geth ]
when:
- "'el' in node"
- node.el == "geth"
with_items: '{{ nodes_layout[inventory_hostname] }}'
loop_control: { loop_var: node, index_var: idx }
- include_role:
name: infra-role-geth-exporter
apply:
tags: always
tags: [ geth-exporter ]
when:
- "'el' in node"
- node.el == "geth"
with_items: '{{ nodes_layout[inventory_hostname] }}'
loop_control: { loop_var: node, index_var: idx }
- name: Deploy Mainnet Erigon Nodes
become: true
serial: '{{ serial|default(1) }}'
hosts: nimbus-mainnet-erigon
vars_files: layout/mainnet.yml
tasks:
- include_role:
name: infra-role-erigon
apply:
tags: always
tags: [ erigon ]
when:
- "'el' in node"
- node.el == "erigon"
with_items: '{{ nodes_layout[inventory_hostname] }}'
loop_control: { loop_var: node, index_var: idx }
- name: Deploy Mainnet Nimbus ETH1 Nodes
become: true
serial: '{{ serial|default(1) }}'
hosts: nimbus-mainnet-nel
vars_files: layout/mainnet.yml
tasks:
- include_role:
name: infra-role-nimbus-eth1
apply:
tags: always
tags: [ nimbus-eth1 ]
when:
- "'el' in node"
- node.el == "nel"
with_items: '{{ nodes_layout[inventory_hostname] }}'
loop_control: { loop_var: node, index_var: idx }

View File

@ -50,11 +50,11 @@
- name: infra-role-nimbus-eth1
src: git@github.com:status-im/infra-role-nimbus-eth1.git
version: a9e82b5193d4930962f6a62e79d3b431a4030823
version: 53e0f5626298b2bf0cd863e4ab87379f08a50074
- name: infra-role-nimbus-fluffy
src: git@github.com:status-im/infra-role-nimbus-fluffy.git
version: 164cd9b1f8bf18cdcc3cce7be43e73743394adaf
version: 9e6a5a17843b82f566781deaa7e3c92bc7ea5b1c
- name: infra-role-portal-bridge
src: git@github.com:status-im/infra-role-portal-bridge.git

View File

@ -16,5 +16,7 @@ function update_yaml() {
for FILE in geth-*.holesky.yml; do update_yaml "${FILE}"; done
for FILE in erigon-*.holesky.yml; do update_yaml "${FILE}"; done
for FILE in neth-*.holesky.yml; do update_yaml "${FILE}"; done
for FILE in linux-*.mainnet.yml; do update_yaml "${FILE}"; done
for FILE in nel-*.mainnet.yml; do update_yaml "${FILE}"; done
for FILE in erigon-*.mainnet.yml; do update_yaml "${FILE}"; done
for FILE in geth-*.mainnet.yml; do update_yaml "${FILE}"; done
for FILE in linux-*.sepolia.yml; do update_yaml "${FILE}"; done

View File

@ -9,45 +9,44 @@ nodes_layout:
- { branch: 'stable', history: 'archive' }
# Innova Hosting ---------------------------------------------------
'linux-01.ih-eu-mda1.nimbus.mainnet': # Frankenstein host, all nodes use one Geth.
- { branch: 'stable', num: 1 }
- { branch: 'testing', num: 1, open_libp2p_ports: false }
- { branch: 'unstable', num: 1, public_api: true }
- { branch: 'unstable', num: 2 }
'nel-01.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'nel' }
- { branch: 'testing' }
- { branch: 'unstable', el: 'nel' }
- { branch: 'libp2p' }
'linux-02.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1 }
- { branch: 'testing', num: 1, public_api: true }
- { branch: 'unstable', num: 1, open_libp2p_ports: false }
- { branch: 'unstable', num: 2 }
'nel-02.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'nel' }
- { branch: 'testing', }
- { branch: 'unstable', el: 'nel' }
- { branch: 'libp2p' }
'linux-03.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1, max_peers: 10000 }
- { branch: 'testing', num: 1 }
- { branch: 'unstable', num: 1 }
- { branch: 'unstable', num: 2 }
'nel-03.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'nel' }
- { branch: 'testing' }
- { branch: 'unstable', el: 'nel' }
- { branch: 'libp2p' }
'linux-04.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1 }
- { branch: 'testing', num: 1 }
- { branch: 'unstable', num: 1 }
- { branch: 'unstable', num: 2 }
'erigon-01.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'erigon' }
- { branch: 'testing' }
- { branch: 'unstable', el: 'erigon' }
- { branch: 'libp2p' }
'linux-05.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1 }
- { branch: 'testing', num: 1 }
- { branch: 'unstable', num: 1 }
- { branch: 'unstable', num: 2 }
'erigon-02.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'erigon' }
- { branch: 'testing' }
- { branch: 'unstable', el: 'erigon' }
- { branch: 'libp2p' }
'linux-06.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1 }
- { branch: 'testing', num: 1 }
- { branch: 'unstable', num: 1 }
- { branch: 'unstable', num: 2 }
'geth-01.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', el: 'geth', max_peers: 10000 }
- { branch: 'testing' }
- { branch: 'unstable', el: 'geth', public_api: true }
- { branch: 'libp2p' }
'linux-07.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable', num: 1 }
- { branch: 'libp2p', num: 1 }
- { branch: 'libp2p', num: 2 }
- { branch: 'libp2p', num: 3, extra_flags: {'debug-enable-yamux': true} }
- { branch: 'libp2p', num: 4, extra_flags: {'debug-enable-yamux': true} }
'geth-02.ih-eu-mda1.nimbus.mainnet':
- { branch: 'stable' }
- { branch: 'testing', el: 'geth', public_api: true } # we need one testing node with EL for public_api
- { branch: 'unstable', el: 'geth' }
- { branch: 'libp2p' }

2
era.tf
View File

@ -11,7 +11,7 @@ resource "cloudflare_record" "era_sepolia" {
resource "cloudflare_record" "era_mainnet" {
zone_id = local.zones["nimbus.team"]
name = "mainnet.era"
value = module.nimbus_nodes_mainnet_innova.public_ips[2]
value = module.nimbus_nodes_mainnet_innova_erigon.public_ips[0]
type = "A"
proxied = true
}

View File

@ -2,24 +2,52 @@
* CPU: Intel Xeon E5-2690 v2 @ 3.00GHz
* MEM: 62 GB DDR3
* SSD: 1x400 GB, 2x1.6 TB */
module "nimbus_nodes_mainnet_innova" {
module "nimbus_nodes_mainnet_innova_nel" {
source = "github.com/status-im/infra-tf-dummy-module"
name = "linux"
name = "nel"
env = "nimbus"
stage = "mainnet"
group = "nimbus-mainnet-metal"
group = "nimbus-mainnet-nel"
region = "eu-mda1"
prefix = "ih"
ips = [
"194.33.40.70", /* linux-01.ih-eu-mda1.nimbus.mainnet */
"194.33.40.72", /* linux-02.ih-eu-mda1.nimbus.mainnet */
"194.33.40.78", /* linux-03.ih-eu-mda1.nimbus.mainnet */
"194.33.40.99", /* linux-04.ih-eu-mda1.nimbus.mainnet */
"194.33.40.100", /* linux-05.ih-eu-mda1.nimbus.mainnet */
"194.33.40.101", /* linux-06.ih-eu-mda1.nimbus.mainnet */
"185.181.229.82", /* linux-07.ih-eu-mda1.nimbus.mainnet */
"194.33.40.70", /* nel-01.ih-eu-mda1.nimbus.mainnet */
"194.33.40.72", /* nel-02.ih-eu-mda1.nimbus.mainnet */
"194.33.40.78", /* nel-03.ih-eu-mda1.nimbus.mainnet */
]
}
module "nimbus_nodes_mainnet_innova_erigon" {
source = "github.com/status-im/infra-tf-dummy-module"
name = "erigon"
env = "nimbus"
stage = "mainnet"
group = "nimbus-mainnet-erigon"
region = "eu-mda1"
prefix = "ih"
ips = [
"194.33.40.99", /* erigon-01.ih-eu-mda1.nimbus.mainnet */
"194.33.40.100", /* erigon-02.ih-eu-mda1.nimbus.mainnet */
]
}
module "nimbus_nodes_mainnet_innova_geth" {
source = "github.com/status-im/infra-tf-dummy-module"
name = "geth"
env = "nimbus"
stage = "mainnet"
group = "nimbus-mainnet-geth"
region = "eu-mda1"
prefix = "ih"
ips = [
"194.33.40.101", /* geth-01.ih-eu-mda1.nimbus.mainnet */
"185.181.229.82", /* geth-02.ih-eu-mda1.nimbus.mainnet */
]
}
@ -27,7 +55,7 @@ module "nimbus_nodes_mainnet_innova" {
resource "cloudflare_record" "unstable_mainnet_beacon_api" {
zone_id = local.zones["nimbus.team"]
name = "unstable.mainnet.beacon-api"
value = module.nimbus_nodes_mainnet_innova.public_ips[0]
value = module.nimbus_nodes_mainnet_innova_geth.public_ips[0]
type = "A"
proxied = false
}
@ -35,7 +63,7 @@ resource "cloudflare_record" "unstable_mainnet_beacon_api" {
resource "cloudflare_record" "testing_mainnet_beacon_api" {
zone_id = local.zones["nimbus.team"]
name = "testing.mainnet.beacon-api"
value = module.nimbus_nodes_mainnet_innova.public_ips[1]
value = module.nimbus_nodes_mainnet_innova_geth.public_ips[1]
type = "A"
proxied = false
}

View File

@ -12,7 +12,9 @@ output "hosts" {
module.nimbus_nodes_holesky_innova_neth.hosts,
module.nimbus_nodes_holesky_innova_macm2.hosts,
module.nimbus_nodes_holesky_innova_windows.hosts,
module.nimbus_nodes_mainnet_innova.hosts,
module.nimbus_nodes_mainnet_innova_nel.hosts,
module.nimbus_nodes_mainnet_innova_erigon.hosts,
module.nimbus_nodes_mainnet_innova_geth.hosts,
module.nimbus_nodes_mainnet_stable_small.hosts,
module.nimbus_nodes_sepolia_innova.hosts,
)