mainnet/prater: block libp2p ports for select nodes

As part of experiment in how nodes behave in restrictive networks.

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/group_vars/nimbus.mainnet.yml

@@ -2,10 +2,79 @@
 swap_file_size_mb: 2048
 
 beacon_node_network: 'mainnet'
-beacon_node_repo_branch: 'stable'
+beacon_node_repo_branch: '{{ node.branch }}'
-
+# TODO Drop this once all nodes have an number.
+service_number: '{{ node.get("num", False) | ternary(("-%02d"|format(node.get("num", 0))), "") }}'
+beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch }}{{ service_number }}'
+# Ports
+beacon_node_discovery_port: '{{ 9000 + port_offset }}'
+beacon_node_listening_port: '{{ 9000 + port_offset }}'
+beacon_node_metrics_port:   '{{ 9200 + port_offset }}'
+beacon_node_rest_port:      '{{ 9300 + port_offset }}'
+beacon_node_rpc_port:       '{{ 9900 + port_offset }}'
+# Firewall
+beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
+# Builds
+beacon_node_build_frequency: '*-*-* {{ 12 + port_offset }}:00:00'
+# Tuning
+beacon_node_threads: '{{ (node.branch == "testing") | ternary(4, 1) }}'
 # Mainnet validators run on a separate fleet.
 beacon_node_dist_validators_enabled: false
-
 # Bootnodes should subscribe to all subnets
 beacon_node_subscribe_all: true
+
+# Split by hostname for more central location
+nodes_layout:
+  'stable-small-01.aws-eu-central-1a.nimbus.mainnet':
+    - { branch: 'stable' }
+
+  'stable-small-02.aws-eu-central-1a.nimbus.mainnet':
+    - { branch: 'stable' }
+
+  'metal-01.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   num: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1, open_libp2p_ports: false }
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1 } # Public Beacon API
+    - { branch: 'unstable', num: 2 }
+
+  'metal-02.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   nun: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1 } # Public Beacon API
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1, open_libp2p_ports: false }
+    - { branch: 'unstable', num: 2 }
+
+  'metal-03.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   num: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1 }
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1 }
+    - { branch: 'unstable', num: 2 }
+
+  'metal-04.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   num: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1 }
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1 }
+    - { branch: 'unstable', num: 2 }
+
+  'metal-05.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   num: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1 }
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1 }
+    - { branch: 'unstable', num: 2 }
+
+  'metal-06.he-eu-hel1.nimbus.mainnet':
+    - { branch: 'stable',   num: 1 }
+    - { branch: 'stable',   num: 2 }
+    - { branch: 'testing',  num: 1 }
+    - { branch: 'testing',  num: 2 }
+    - { branch: 'unstable', num: 1 }
+    - { branch: 'unstable', num: 2 }

ansible/group_vars/nimbus.prater.yml

@@ -13,7 +13,7 @@ beacon_node_metrics_port:   '{{ 9200 + port_offset }}'
 beacon_node_rest_port:      '{{ 9300 + port_offset }}'
 beacon_node_rpc_port:       '{{ 9900 + port_offset }}'
 # Firewall
-beacon_node_firewall_libp2p_open: '{{ node.get("block_libp2p_ports") is defined | ternary(false, true) }}'
+beacon_node_firewall_libp2p_open: '{{ node.get("open_libp2p_ports", true) }}'
 # Tuning
 beacon_node_max_peers: 300
 beacon_node_threads: '{{ (node.branch == "testing") | ternary(4, 1) }}'
@@ -51,17 +51,17 @@ nodes_layout:
   'metal-01.he-eu-hel1.nimbus.prater':
     - { branch: 'stable',   num: 1, start: 10000, end: 12500, build_freq: '*-*-* 12:00:00' }
     - { branch: 'stable',   num: 2,                           build_freq: '*-*-* 13:00:00' }
-    - { branch: 'testing',  num: 1, start: 22500, end: 25000, build_freq: '*-*-* 14:00:00' }
+    - { branch: 'testing',  num: 1, start: 22500, end: 25000, build_freq: '*-*-* 14:00:00', open_libp2p_ports: false }
     - { branch: 'testing',  num: 2,                           build_freq: '*-*-* 15:00:00' }
-    - { branch: 'unstable', num: 1, start: 35000, end: 37500, build_freq: '*-*-* 16:00:00' }
+    - { branch: 'unstable', num: 1, start: 35000, end: 37500, build_freq: '*-*-* 16:00:00', open_libp2p_ports: false }
-    - { branch: 'unstable', num: 2,                           build_freq: '*-*-* 17:00:00' }
+    - { branch: 'unstable', num: 2,                           build_freq: '*-*-* 17:00:00' } # Public Beacon API
 
   # 2500 validators per first node
   'metal-02.he-eu-hel1.nimbus.prater':
     - { branch: 'stable',   num: 1, start:  5000, end:  7500, build_freq: '*-*-* 12:00:00' }
     - { branch: 'stable',   num: 2,                           build_freq: '*-*-* 13:00:00' }
     - { branch: 'testing',  num: 1, start: 17500, end: 20000, build_freq: '*-*-* 14:00:00' }
-    - { branch: 'testing',  num: 2,                           build_freq: '*-*-* 15:00:00' }
+    - { branch: 'testing',  num: 2,                           build_freq: '*-*-* 15:00:00' } # Public Beacon API
     - { branch: 'unstable', num: 1, start: 30000, end: 32500, build_freq: '*-*-* 16:00:00' }
     - { branch: 'unstable', num: 2,                           build_freq: '*-*-* 17:00:00' }
 

ansible/mainnet.yml

@@ -17,8 +17,15 @@
     beacon_node_netkey: '{{lookup("bitwarden", "nimbus/netkey", field=hostname)}}'
   roles:
     - { role: swap-file,                    tags: [ swap-file ] }
+    - { role: redirect-ports,               tags: [ redirect-ports ] }
     - { role: get-geth-web3-urls,           tags: [ get-geth-web3-urls ] }
-    - { role: infra-role-beacon-node-linux, tags: [ beacon-node ] }
+  tasks:
+    - include_role: name=infra-role-beacon-node-linux
+      tags: [ beacon-node, infra-role-beacon-node-linux ]
+      with_items: '{{ nodes_layout[hostname] }}'
+      loop_control:
+        loop_var: node
+        index_var: port_offset
 
 - name: Deploy mainnet linux beacon nodes
   become: true
@@ -28,25 +35,8 @@
     - { role: get-geth-web3-urls,           tags: [ get-geth-web3-urls ] }
   tasks:
     - include_role: name=infra-role-beacon-node-linux
-      tags: beacon-node
+      tags: [ beacon-node, infra-role-beacon-node-linux ]
-      vars:
+      with_items: '{{ nodes_layout[hostname] }}'
-        beacon_node_service_name: 'beacon-node-{{ beacon_node_network }}-{{ node.branch }}-{{ "%02d"|format(node.idx) }}'
-        beacon_node_repo_branch: '{{ node.branch }}'
-        beacon_node_threads: '{{ (node.branch == "testing") | ternary(4, 1) }}'
-        beacon_node_build_frequency: '{{ node.build_frequency }}'
-        beacon_node_open_firewall: '{{ node.open_firewall | default(true) }}'
-        beacon_node_discovery_port: '{{ 9000 + port_offset }}'
-        beacon_node_listening_port: '{{ 9000 + port_offset }}'
-        beacon_node_metrics_port:   '{{ 9200 + port_offset }}'
-        beacon_node_rest_port:      '{{ 9300 + port_offset }}'
-        beacon_node_rpc_port:       '{{ 9900 + port_offset }}'
-      with_items:
-        - { branch: 'stable',   idx: 1, build_frequency: '*-*-* 12:00:00' }
-        - { branch: 'stable',   idx: 2, build_frequency: '*-*-* 13:00:00' }
-        - { branch: 'testing',  idx: 1, build_frequency: '*-*-* 14:00:00' }
-        - { branch: 'testing',  idx: 2, build_frequency: '*-*-* 15:00:00' }
-        - { branch: 'unstable', idx: 1, build_frequency: '*-*-* 16:00:00' }
-        - { branch: 'unstable', idx: 2, build_frequency: '*-*-* 17:00:00' }
       loop_control:
         loop_var: node
         index_var: port_offset

ansible/requirements.yml

@@ -46,7 +46,7 @@
 
 - name: infra-role-beacon-node-linux
   src: git@github.com:status-im/infra-role-beacon-node-linux.git
-  version: 31abbf8d8c019b024487de903d9b5237c1973519
+  version: 3af4e926c1a017335747154a5e9a654ac04ecf0e
   scm: git
 
 - name: infra-role-beacon-node-windows