infra-nimbus/ansible/group_vars/nimbus.prater.yml

---
# Go-Ethereum
geth_service_name: 'geth-goerli-{{ "%02d"|format(idx|int+1) }}'
geth_service_path: '/docker/{{ geth_service_name }}'
geth_network_name: 'goerli'
geth_cont_name: '{{ geth_service_name }}-node'
geth_cont_vol: '{{ geth_service_path }}/node'
geth_sync_mode: 'snap'
geth_log_level_name: info
geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
# Memory settings¬
geth_cont_mem_ratio: 0.10
geth_cache_size: '{{ (ansible_memtotal_mb * 0.05|float) | int }}'
# Ports
geth_rpc_enabled: true
geth_rpc_addr: '0.0.0.0'
geth_rpc_port:     '{{ 8545 + (idx|int) + 1 }}'
geth_port:         '{{ 30303 + (idx|int) + 1 }}'
geth_metrics_port: '{{ 6060 + (idx|int) + 1 }}'
geth_authrpc_port: '{{ 8551 + (idx|int) + 1 }}'
# Geth metrics¬
geth_expo_service_name: '{{ geth_service_name }}'
geth_expo_source_cont_name: '{{ geth_cont_name }}'
geth_expo_source_data_path: '{{ geth_cont_vol }}/data'
geth_expo_cont_port: '{{ 9400 + (idx|int) + 1 }}'

# Nethermind
nethermind_network_name: 'goerli'
nethermind_service_name: 'nethermind-{{ nethermind_network_name }}-{{ "%02d"|format(idx|int+1) }}'
nethermind_service_path: '/docker/{{ nethermind_service_name }}'
nethermind_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
nethermind_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
nethermind_sync_mode: 'snap'
nethermind_port:         '{{ 40303 + (idx|int) + 1 }}'
nethermind_rpc_port:     '{{ 8645  + (idx|int) + 1 }}'
nethermind_authrpc_port: '{{ 8651  + (idx|int) + 1 }}'
nethermind_metrics_port: '{{ 6660  + (idx|int) + 1 }}'

# Erigon node
erigon_network_name: 'goerli'
erigon_service_name: 'erigon-{{ erigon_network_name }}-{{ "%02d"|format(idx|int+1) }}'
erigon_prune: 'hrtc' # TODO: Possibly wrong, verify.
erigon_log_level: 'info'
erigon_cont_mem_ratio: 0.15
erigon_max_peers: 20
erigon_miner_enabled: true
erigon_mining_private_key: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="private-key")}}'
erigon_discovery_dns_entries: ['enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.goerli.ethdisco.net']
# Ports
erigon_port: '{{ 40303 + (idx|int) + 1 }}'
erigon_metrics_enabled: true
erigon_metrics_port: '{{ 7060 + (idx|int) + 1 }}'
erigon_rpc_addr: '0.0.0.0'
erigon_rpc_port: '{{ 9545 + (idx|int) + 1 }}'
erigon_authrpc_addr: '0.0.0.0'
erigon_authrpc_port: '{{ 9551 + (idx|int) + 1 }}'
erigon_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'

# Nimbus Beacon node
beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch }}'
beacon_node_network: 'prater'
beacon_node_era_dir_path: '{{ nimbus_era_files_timer_path }}'
beacon_node_repo_branch: '{{ node_name_to_branch_map.get(node.branch, node.branch) }}'
# We map short names to branches to avoid too long service names.
node_name_to_branch_map:
  libp2p: 'nim-libp2p-auto-bump-unstable'
# Ports
beacon_node_discovery_port: '{{ 9000 + idx }}'
beacon_node_listening_port: '{{ 9000 + idx }}'
beacon_node_metrics_port:   '{{ 9200 + idx }}'
beacon_node_rest_port:      '{{ beacon_node_rest_port_base + idx }}'
beacon_node_rest_port_base: 9300
beacon_node_rest_address:   '0.0.0.0'
beacon_node_rest_max_body_size: 65536
# Firewall
beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
# Tuning
beacon_node_max_peers: 300
beacon_node_service_nofile_limit: 32768
beacon_node_cores_per_node: '{{ (ansible_processor_vcpus|int / nodes_layout[hostname]|length) | round(0, "ceil") | int }}'
beacon_node_threads: '{{ (node.branch == "libp2p") | ternary(1, beacon_node_cores_per_node) }}'
beacon_node_history_retention: '{{ "archive" if node.get("public_api") else node.get("history", "prune") }}'
beacon_node_extra_flags: '{{ node.extra_flags if node.extra_flags is defined else [] }}'
# Monitoring
beacon_node_validator_monitor_auto: true
beacon_node_validator_monitor_details: >-
  {{  (node.public_api is not defined or not node.public_api)
  and (node.end is defined and (node.end - node.start) <= 64) }}
# Builds
beacon_node_build_nim_flags: '-d:noSignalHandler {{ node.get("nim_flags", "") }}'
beacon_node_build_frequency: '{{ node.get("build_freq", "daily") }}'
beacon_node_build_start_time: '{{ node.get("build_start", omit) }}'
# Validators from nimbus-private repo
beacon_node_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}'
beacon_node_dist_validators_start: '{{ node.get("validator_client", false) | ternary(0, node.start) | mandatory }}'
beacon_node_dist_validators_end:   '{{ node.get("validator_client", false) | ternary(0, node.end)   | mandatory }}'
# Suggests it to the Execution Layer client and the builder network.
beacon_node_suggested_fee_recipient: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="address")}}'
# Windows service user
beacon_node_service_user_pass: '{{lookup("bitwarden", "nimbus/windows", field="password")}}'
# MEV Payload Builder
beacon_node_payload_builder_enabled: '{{ node.get("payload_builder", false) }}'
beacon_node_payload_builder_url: '{{ prater_mev_boost_urls[idx] }}'
prater_mev_boost_urls:
  - 'https://bloxroute.max-profit.builder.goerli.blxrbdn.com'
  - 'https://builder-relay-goerli.blocknative.com'
  - 'https://goerli-relay.securerpc.com'
  - 'https://goerli.aestus.live'
  - 'https://relay-stag.ultrasound.money'
# Light client data
beacon_node_light_client_data_enabled: '{{ (node.public_api is defined and node.public_api) }}'
beacon_node_light_client_data_serve: true
beacon_node_light_client_data_import_mode: 'full'
# Execution layer Enginer API
beacon_node_exec_layer_urls: >-
  {{ [ "http://localhost:"+geth_authrpc_port ] +
    ([ "http://localhost:"+erigon_authrpc_port ]     if node.get("erigon", False)     else []) +
    ([ "http://localhost:"+nethermind_authrpc_port ] if node.get("nethermind", False) else [])
  }}
beacon_node_exec_layer_jwt_secret: '{{ geth_authrpc_jwtsecret }}'
# Periodic resync to save space
beacon_node_resync_enabled: true
beacon_node_resync_timer_enabled: '{{ (idx % 2 == 1) and not node.get("public_api") }}'
beacon_node_resync_timer_frequency: 'monthly'
beacon_node_resync_timer_random_delay_sec: 604800 # 7 days
beacon_node_resync_timer_trusted_api_url: >-
  {{ "http://stable-large-01.aws-eu-central-1a.nimbus.prater.wg:9300"
     if idx == 0  else "http://localhost:"+(beacon_node_rest_port_base|string) }}

# Validator Client
validator_client_service_name: 'validator-client-{{ validator_client_network }}-{{ node.branch }}'
validator_client_service_enabled: '{{ node.get("validator_client", false) }}'
validator_client_network: '{{ beacon_node_network }}'
validator_client_log_level: 'DEBUG'
validator_client_beacon_node_url: 'http://127.0.0.1:{{ beacon_node_rest_port }}'
validator_client_doppelganger_detection: false
validator_client_service_nofile_limit: 32768
# Builds
validator_client_build_repo_branch: '{{ "unstable" if "libp2p" in beacon_node_repo_branch else beacon_node_repo_branch  }}'
validator_client_build_frequency: 'daily'
# Ports
validator_client_metrics_port:    '{{ 8108 + idx|int + 1 }}'
validator_client_keymanager_port: '{{ 5052 + idx|int + 1 }}'
# Suggests it to the Execution Layer client and the builder network.
validator_client_suggested_fee_recipient: '{{lookup("bitwarden", "nimbus/wallet/testnets", field="address")}}'
# MEV Payload Builder
validator_client_payload_builder_enabled: '{{ node.get("payload_builder", false) }}'
# Keymanager
validator_client_keymanager_enabled: true
validator_client_keymanager_token: '{{lookup("bitwarden", "nimbus/keymanager", field="token")}}'
# Validators Distribution
validator_client_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}'
validator_client_dist_validators_start: '{{ (not node.get("validator_client", false)) | ternary(0, node.start) | mandatory }}'
validator_client_dist_validators_end:   '{{ (not node.get("validator_client", false)) | ternary(0, node.end)   | mandatory }}'

# ERA files geneartion.
nimbus_era_files_timer_enabled: '{{ (nodes_layout[hostname]|length) > 1 }}'
nimbus_era_files_timer_path: '/data/era'
nimbus_era_files_network: '{{ beacon_node_network }}'
# FIXME: Not pretty, since hardcoded, but the simplest way to do it right now.
nimbus_era_files_db_path: '/data/beacon-node-{{ beacon_node_network }}-stable/data/db'
# FIXME: Using unstable version to include fixes not present in stable.
nimbus_era_files_nclidb_path: '/data/beacon-node-{{ beacon_node_network }}-unstable/repo/build/ncli_db'

# Open Ports
open_ports_default_comment: 'Nimbus REST API'
open_ports_default_chain: 'VPN'
open_ports_list:
  - { port: '9300:9310', ipset: '{{ env }}.{{ stage }}' }
  - { port: '9400:9404', ipset: 'metrics.hq',  comment: 'Geth Exporter' }
  - { port: '7060:7070', ipset: 'metrics.hq',  comment: 'Erigon Metrics' }
  - { port: '{{ smart_metrics_listen_port }}', ipset: 'metrics.hq', comment: 'SMART Metrics' }
  # https://github.com/status-im/infra-nim-waku/issues/59
  - { port: '9546:9550', ipset: 'wakuv2.test', comment: 'Wakuv2 Websocket' }

# Split by hostname for more central location
nodes_layout:
  # WARNING: The nodes hosted on AWS are bootstrap nodes and should not be changed.
  'stable-large-01.aws-eu-central-1a.nimbus.prater': # 2000 each
    - { branch: 'stable',   start:     0, end:  2000, history: 'archive' }

  'testing-large-01.aws-eu-central-1a.nimbus.prater': # 2000 each
    - { branch: 'testing',  start:  2000, end:  4000, history: 'archive' }

  'unstable-large-01.aws-eu-central-1a.nimbus.prater': # 2000 each
    - { branch: 'unstable', start:  4000, end:  6000, history: 'archive', payload_builder: true }

  'macos-01.ms-eu-dublin.nimbus.prater': # 3000 each
    - { branch: 'testing',   start:  6000, end:  9000, build_start: '13:00:00' }
    - { branch: 'unstable',  start:  9000, end: 10000, build_start: '15:00:00' }

  'windows-01.he-eu-hel1.nimbus.prater': # 2000 each
    - { branch: 'stable',   start: 14000, end: 16000, build_start: '13:00:00' }
    - { branch: 'testing',  start: 16000, end: 18000, build_start: '15:00:00' }
    - { branch: 'unstable', start: 18000, end: 20000, build_start: '16:00:00', nim_flags: '-d:json_rpc_websocket_package=websock' }

  'windows-01.ih-eu-mda1.nimbus.prater':
    - { branch: 'stable',                             build_start: '13:00:00' }
    - { branch: 'testing',                            build_start: '15:00:00' }
    - { branch: 'unstable',                           build_start: '16:00:00', nim_flags: '-d:json_rpc_websocket_package=websock' }

  # Innova Hosting
  'linux-01.ih-eu-mda1.nimbus.prater': # 0 each
    - { branch: 'stable',                             build_freq: '*-*-* 13:00:00' }
    - { branch: 'testing',                            build_freq: '*-*-* 15:00:00' }
    - { branch: 'unstable',                           build_freq: '*-*-* 17:00:00', public_api: true }
    - { branch: 'libp2p',                             build_freq: '*-*-* 19:00:00' }

  'linux-02.ih-eu-mda1.nimbus.prater': # 1 each
    - { branch: 'stable',   start: 20000, end: 20001, build_freq: '*-*-* 13:00:00' }
    - { branch: 'testing',  start: 20001, end: 20002, build_freq: '*-*-* 15:00:00', public_api: true }
    - { branch: 'unstable', start: 20002, end: 20003, build_freq: '*-*-* 17:00:00', nethermind: true }
    - { branch: 'libp2p',   start: 20003, end: 20004, build_freq: '*-*-* 19:00:00' }

  'linux-03.ih-eu-mda1.nimbus.prater': # 10 each
    - { branch: 'stable',   start: 20004, end: 20014, build_freq: '*-*-* 11:00:00' }
    - { branch: 'testing',  start: 20024, end: 20034, build_freq: '*-*-* 15:00:00' }
    - { branch: 'unstable', start: 20014, end: 20024, build_freq: '*-*-* 13:00:00', validator_client: true, open_libp2p_ports: false }
    - { branch: 'libp2p',   start: 20034, end: 20044, build_freq: '*-*-* 17:00:00' }

  'linux-04.ih-eu-mda1.nimbus.prater': # 30 each
    - { branch: 'stable',   start: 20044, end: 20074, build_freq: '*-*-* 11:00:00', validator_client: true }
    - { branch: 'testing',  start: 20104, end: 20134, build_freq: '*-*-* 15:00:00' }
    - { branch: 'unstable', start: 20074, end: 20104, build_freq: '*-*-* 13:00:00', payload_builder: true }
    - { branch: 'libp2p',   start: 20134, end: 20164, build_freq: '*-*-* 17:00:00' }

  'linux-05.ih-eu-mda1.nimbus.prater': # 60 each
    - { branch: 'stable',   start: 20164, end: 20224, build_freq: '*-*-* 11:00:00' }
    - { branch: 'testing',  start: 20284, end: 20344, build_freq: '*-*-* 15:00:00' }
    - { branch: 'unstable', start: 20224, end: 20284, build_freq: '*-*-* 13:00:00', open_libp2p_ports: false, erigon: true }
    - { branch: 'libp2p',   start: 20344, end: 20404, build_freq: '*-*-* 17:00:00', extra_flags: ['--debug-use-old-attestation-stability-subnets=false']  }

  'linux-06.ih-eu-mda1.nimbus.prater':
    - { branch: 'stable',   start: 20404, end: 31303, build_freq: '*-*-* 11:00:00' } # 10899 validators
    - { branch: 'testing',  start: 31303, end: 39202, build_freq: '*-*-* 15:00:00', validator_client: true, payload_builder: true, open_libp2p_ports: false } # 7899 validators
    - { branch: 'unstable', start: 39202, end: 45101, build_freq: '*-*-* 13:00:00', payload_builder: true, erigon: true } # 5899 validators
    - { branch: 'libp2p',   start: 45101, end: 49617, build_freq: '*-*-* 17:00:00', validator_client: true } # 4516 validators