infra-nimbus/ansible/group_vars/nimbus.mainnet.yml

---
# Beacon nodes are held on /data so we can't bind it.
bootstrap__extra_volume_path: '/docker'
bootstrap__extra_volume_bind_path: null

# SWAP
swap_file_size_mb: 2048

# Ports used for all 3 EL nodes as they are mutually exclusive.
exec_layer_p2p_port:     '{{ 30303 + 3*idx|int + 1 }}'
exec_layer_rpc_port:     '{{ 8545  +   idx|int + 1 }}'
exec_layer_authrpc_port: '{{ 8551  +   idx|int + 1 }}'
exec_layer_metrics_port: '{{ 6060  +   idx|int + 1 }}'

# Go-Ethereum ------------------------------------------------------------------

geth_service_name: 'geth-{{ geth_network_name }}-{{ node.branch | mandatory }}'
geth_service_path: '/docker/{{ geth_service_name }}'
geth_network_name: 'mainnet'
geth_cont_name: '{{ geth_service_name }}-node'
geth_cont_vol: '{{ geth_service_path }}/node'
geth_sync_mode: 'snap'
geth_log_level_name: 'info'
# Geth auth & JWT token
geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
geth_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'
# Memory settings
geth_cont_mem_ratio: 0.15
geth_cache_size: '{{ (ansible_memtotal_mb * 0.05|float) | int }}'
# Ports
geth_rpc_addr:        '0.0.0.0'
geth_authrpc_addr:    '127.0.0.1'
geth_port:            '{{ exec_layer_p2p_port }}'
geth_rpc_port:        '{{ exec_layer_rpc_port }}'
geth_authrpc_port:    '{{ exec_layer_authrpc_port }}'
geth_metrics_port:    '{{ exec_layer_metrics_port }}'
# Geth metrics
geth_expo_service_name: '{{ geth_service_name }}'
geth_expo_source_cont_name: '{{ geth_cont_name }}'
geth_expo_source_data_path: '{{ geth_cont_vol }}/data'
geth_expo_cont_port:  '{{ 9400 + (idx|int) + 1 }}'

# Erigon -----------------------------------------------------------------------

erigon_network_name: 'mainnet'
erigon_service_name: 'erigon-{{ geth_network_name }}-{{ node.branch | mandatory }}'
erigon_prune: 'hrtc' # TODO: Possibly wrong, verify.
erigon_log_level: 'info'
erigon_cont_mem_ratio: 0.2
erigon_max_peers: 20
erigon_miner_enabled: true
# Ports
erigon_metrics_enabled: true
erigon_rpc_addr: '0.0.0.0'
erigon_authrpc_addr: '0.0.0.0'
erigon_port:              '{{ exec_layer_p2p_port }}'
erigon_p2p_allowed_ports: ['{{ erigon_port }}', '{{ erigon_port|int + 1 }}']
erigon_rpc_port:          '{{ exec_layer_rpc_port }}'
erigon_metrics_port:      '{{ exec_layer_metrics_port }}'
erigon_authrpc_port:      '{{ exec_layer_authrpc_port }}'
erigon_authrpc_jwtsecret: '{{ beacon_node_exec_layer_jwt_secret }}'

# Nimbus ETH1 ------------------------------------------------------------------

nimbus_eth1_service_name: 'nimbus-eth1-{{ geth_network_name }}-{{ node.branch | mandatory }}'
nimbus_eth1_repo_branch: 'master'
nimbus_eth1_network: 'mainnet'
nimbus_eth1_max_peers: 160
nimbus_eth1_log_level: 'DEBUG'
nimbus_eth1_service_path: '/docker/{{ nimbus_eth1_service_name }}'
nimbus_eth1_engine_http_enabled: true
nimbus_eth1_engine_ws_enabled: true
nimbus_eth1_era_dir: '{{ nimbus_era_files_timer_path }}'
nimbus_eth1_era1_dir: '/docker/era1'
# Ports
nimbus_eth1_listening_port:   '{{ exec_layer_p2p_port }}'
nimbus_eth1_discovery_port:   '{{ exec_layer_p2p_port }}'
nimbus_eth1_http_port:        '{{ exec_layer_rpc_port }}'
nimbus_eth1_metrics_port:     '{{ exec_layer_metrics_port }}'
nimbus_eth1_engine_http_port: '{{ exec_layer_authrpc_port }}'
nimbus_eth1_metrics_address: '0.0.0.0'
# API secert
nimbus_eth1_jwt_secret: '{{ beacon_node_exec_layer_jwt_secret }}'

# Beacon Nodes -----------------------------------------------------------------

beacon_node_network: 'mainnet'
beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch | mandatory }}'
beacon_node_service_path: '/data/{{ beacon_node_service_name }}'
beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}'
# Tuning
beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else {} }}'
beacon_node_cores_per_node: '{{ (ansible_processor_vcpus / nodes_layout[inventory_hostname]|length) | round(0, "ceil") | int }}'
beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
# We map short names to branches to avoid too long service names.
node_name_to_branch_map:
  libp2p: 'nim-libp2p-auto-bump-unstable'
# Builds
beacon_node_update_build_targets: ['nimbus_beacon_node', 'ncli_db']
beacon_node_update_build: '{{ beacon_node_repo_branch != "stable" }}'
beacon_node_update_frequency: '*-*-* {{ "%02d" | format(idx * 2) }}:00:00'
# Monitoring
beacon_node_validator_monitor_auto: true
beacon_node_validator_monitor_details: '{{ node.public_api is not defined or not node.public_api }}'
# Execution layer Enginer API
beacon_node_exec_layer_jwt_secret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
beacon_node_exec_layer_urls_local: ['http://localhost:{{ exec_layer_authrpc_port }}']
beacon_node_exec_layer_urls: '{{ beacon_node_exec_layer_urls_local if node.get("el") else [] }}'
# Light client data
beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
beacon_node_light_client_data_serve: true
beacon_node_light_client_data_import_mode: 'full'
# Mainnet validators run on a separate fleet.
beacon_node_dist_validators_enabled: false
# Ports
beacon_node_rest_port_base: 9300
beacon_node_rest_port:      '{{ beacon_node_rest_port_base + idx|int + 1 }}'
beacon_node_discovery_port: '{{ 9000 + idx|int + 1 }}'
beacon_node_listening_port: '{{ 9000 + idx|int + 1 }}'
beacon_node_metrics_port:   '{{ 9200 + idx|int + 1 }}'
beacon_node_rest_address:   '0.0.0.0'
beacon_node_max_peers: '{{ node.get("max_peers", 320) }}'
# Firewall
beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
# Excellent stress test and good service to the community.
beacon_node_subscribe_all: true
# FIXME: Temporary test to debug REST API timeout issues.
# https://github.com/status-im/nimbus-eth2/issues/5838
#beacon_node_rest_allow_origin: '{{ "*" if node.get("public_api") else "" }}'

# Reduce Consul alerts sensitivity
beacon_node_consul_check_disabled: '{{ node.get("public_api", false) }}'
beacon_node_consul_check_interval: '60s'
beacon_node_consul_check_timeout: '10s'
beacon_node_consul_success_before_passing: '{{ 5 if node.get("public_api") else 60 }}' # 1h by default
beacon_node_consul_failures_before_warning:  120 # 2h
beacon_node_consul_failures_before_critical: 180 # 3h

# ERA files geneartion.
nimbus_era_files_timer_enabled: '{{ (nodes_layout[inventory_hostname]|length) > 1 }}'
nimbus_era_files_timer_path: '/data/era'
nimbus_era_files_network: '{{ beacon_node_network }}'
# FIXME: Not pretty, since hardcoded, but the simplest way to do it right now.
nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-testing/data/db'
nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-testing/repo/build/ncli_db'

# Migrated to NFTables from IPTables.
# https://github.com/status-im/infra-misc/issues/301
bootstrap__firewall_nftables: true

# Open Ports
host_el_type: '{{ ansible_hostname|split("-")|first }}'
open_ports_list:
  nginx:
    - { port: '443', comment: 'Nginx' }
  smart-metrics:
    - { port: '{{ smart_metrics_listen_port }}', comment: 'SMART Metrics', ipset: 'hq.metrics', iifname: 'wg0' }
  exec-node:
    - { port: '30304-30315', comment: '{{ host_el_type }} LibP2P',    protocol: 'tcp'                     }
    - { port: '30304-30315', comment: '{{ host_el_type }} Discovery', protocol: 'udp'                     }
    - { port: '6061-6064',   comment: '{{ host_el_type }} Metrics',   ipset: 'hq.metrics', iifname: 'wg0' }
    - { port: '9401-9404',   comment: '{{ host_el_type }} Exporter',  ipset: 'hq.metrics', iifname: 'wg0' }
    - { port: '8552-8555',   comment: 'Go-Ethereum AuthRPC',   ipset: '{{ env }}.{{ stage }}', iifname: 'wg0' }
    - { port: '8446-8449',   comment: 'RPC for Portal Bridge', ipset: 'nimbus.fluffy',         iifname: 'wg0' }
  beacon-node:
    - { port: '9001-9004', comment: 'Beacon Node libp2p',    protocol: 'tcp'                                    }
    - { port: '9001-9004', comment: 'Beacon Node discovery', protocol: 'udp'                                    }
    - { port: '9201-9204', comment: 'Beacon Node Metrics',   ipset: 'hq.metrics',                iifname: 'wg0' }
    - { port: '9301-9304', comment: 'Beacon Node REST API',  ipset: '{{ env }}.{{ stage }}',     iifname: 'wg0' }

# Fleet layout can be found in: ansible/vars/layout/mainnet.yml