use Geth Auth RPC endpoints for Engine API

Needs JWT secrets configured and changes to Consul queries.

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/group_vars/nimbus.mainnet.yml

@@ -29,7 +29,7 @@ beacon_node_dist_validators_enabled: false
 # Bootnodes should subscribe to all subnets
 beacon_node_subscribe_all: true
 # HTTP RPC support is unstable
-beacon_node_web3_urls: '{{ beacon_node_web3_urls_all | reject("match", "^http://") }}'
+beacon_node_web3_urls: '{{ beacon_node_web3_urls_all }}'
 # Periodic resync to save space
 beacon_node_resync_timer_enabled: '{{ idx % 2 == 1 }}'
 beacon_node_resync_timer_frequency: 'weekly'

ansible/group_vars/nimbus.prater.yml

@@ -40,7 +40,8 @@ beacon_node_dist_validators_end: '{{ node.end | mandatory }}'
 # Windows service user
 beacon_node_service_user_pass: '{{lookup("bitwarden", "nimbus/windows", field="password")}}'
 # HTTP RPC support is unstable
-beacon_node_web3_urls: '{{ beacon_node_web3_urls_all | reject("match", "^http://") }}'
+beacon_node_web3_urls: '{{ beacon_node_web3_urls_all }}'
+beacon_node_web3_jwt_secret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
 
 # Split by hostname for more central location
 nodes_layout:

ansible/requirements.yml

@@ -46,22 +46,22 @@
 
 - name: infra-role-beacon-node-linux
   src: git@github.com:status-im/infra-role-beacon-node-linux.git
-  version: aaa9c09a720ea1ff0dc52889e888fe45254c4366
+  version: 58fab0c4b2e261131a251a655b5476ab95d05911
   scm: git
 
 - name: infra-role-beacon-node-windows
   src: git@github.com:status-im/infra-role-beacon-node-windows.git
-  version: c1b64c611331f9d1b6f456ffe8a9004dacb3b730
+  version: d24d404206b9817b9b8e0fcd817e375655917b1b
   scm: git
 
 - name: infra-role-beacon-node-macos
   src: git@github.com:status-im/infra-role-beacon-node-macos.git
-  version: 4e50f87201cee38e04995c7056c8ad98a34a26be
+  version: ca2ad38067941e98264597ac9a8429def2e9db33
   scm: git
 
 - name: infra-role-nimbus-eth1
   src: git@github.com:status-im/infra-role-nimbus-eth1.git
-  version: 4de341ed040c7af81fe82abca664337ed107948f
+  version: b3490cde3b77484f32716689af5127d783b436d6
   scm: git
 
 - name: infra-role-nimbus-fluffy

ansible/roles/get-geth-api-urls/README.md

@@ -13,5 +13,6 @@ infura_api_tokens:
  - 'cccccccccccccccccccccccccccccccc'
 ```
 And the more we have the less likely that we hit the Infura threshold.
+Infura endpoints are only used for Mainnet.
 
 The Geth node Consul service is expected to have an `url` in metadata.

ansible/roles/get-geth-api-urls/defaults/main.yml

@@ -9,7 +9,10 @@ eth1_network_name: '{{ eth2_to_eth1_map[eth2_network_name] | mandatory }}'
 geth_node_api_fleet_name: 'nimbus.geth'
 geth_node_api_consul_names:
   - 'geth-{{ eth1_network_name | mandatory }}'
-  - 'openeth-{{ eth1_network_name | mandatory }}'
+
+# No Auth RPC setup yet
+#  - 'openeth-{{ eth1_network_name | mandatory }}'
+#
 geth_api_addresses: []
 # The more the merrier, since we don't want to hit Infura throttling.
 infura_api_tokens:

ansible/roles/get-geth-api-urls/tasks/consul/posix.yml

@@ -4,25 +4,17 @@
     url: '{{ consul_catalog_url }}/datacenters'
   register: data_centers
 
-- name: Find available Geth API RPC services
+- name: Find available Geth API Auth RPC services
   uri:
-    url: '{{ consul_catalog_url }}/service/{{ service_name }}-rpc?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
+    url: '{{ consul_catalog_url }}/service/{{ service_name }}-authrpc?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
   with_items: '{{ data_centers.json }}'
   register: geth_rpc_services
 
-- name: Find available Geth API WebSocket services
-  uri:
-    url: '{{ consul_catalog_url }}/service/{{ service_name }}-ws?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
-  with_items: '{{ data_centers.json }}'
-  register: geth_ws_services
-
 # On Windows json attribute is missing when it's [].
 - name: Extract Geth websocket IP and port
   set_fact:
     geth_api_addresses: |
-      {{ geth_api_addresses +
+      {{ (geth_api_addresses + geth_rpc_services.results)
-      (geth_rpc_services.results
-      + geth_ws_services.results)
       | sum(attribute="json", start=[])
-      | map(attribute='ServiceMeta.url')
+      | map(attribute="ServiceMeta.url")
       | list }}

ansible/roles/get-geth-api-urls/tasks/consul/windows.yml

@@ -6,30 +6,20 @@
     return_content: true
   register: data_centers
 
-- name: Find available Geth API RPC services
+- name: Find available Geth API Auth RPC services
   win_uri:
-    url: '{{ consul_catalog_url }}/service/{{ service_name }}-rpc?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
+    url: '{{ consul_catalog_url }}/service/{{ service_name }}-authrpc?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
     content_type: 'application/json'
     return_content: true
   with_items: '{{ data_centers.json }}'
   register: geth_rpc_services
 
-- name: Find available Geth WebSocket services
-  win_uri:
-    url: '{{ consul_catalog_url }}/service/{{ service_name }}-ws?dc={{ item }}&tag={{ geth_node_api_fleet_name }}'
-    content_type: 'application/json'
-    return_content: true
-  with_items: '{{ data_centers.json }}'
-  register: geth_ws_services
-
 # On Windows json attribute is missing when it's [].
 - name: Extract Geth websocket IP and port
   set_fact:
     geth_api_addresses: |
-      {{ geth_api_addresses +
+      {{ (geth_api_addresses + geth_rpc_services.results)
-      (geth_rpc_services.results
-      + geth_ws_services.results)
       | selectattr("json", "defined")
       | sum(attribute="json", start=[])
-      | map(attribute='ServiceMeta.url')
+      | map(attribute="ServiceMeta.url")
       | list }}

ansible/roles/get-geth-api-urls/tasks/main.yml

@@ -13,5 +13,8 @@
 
 - name: Extract Goerli Geth WebSocket URL
   set_fact:
-    # our nodes first (will be used by default) then an infura node
+    # Our nodes first (will be used by default) then an Infura node.
-    beacon_node_web3_urls_all: "{{ geth_api_addresses + [ selected_infura_url ] }}"
+    # Currently only Mainnet has not had it's merge, so it can use Infura.
+    beacon_node_web3_urls_all: |
+      {{ geth_api_addresses +
+      (eth1_network_name == "mainnet") | ternary([selected_infura_url], []) }}

ansible/vars/nimbus-kiln-metal.yml

@@ -4,8 +4,9 @@ geth_service_name: 'geth-kiln-{{ "%02d"|format(idx|int+1) }}'
 geth_network_name: 'kiln'
 geth_sync_mode: 'snap'
 geth_log_level_name: info
-# Geth auth
+# Geth auth & JWT token
 geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
+geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
 # Memory settings
 geth_cont_mem_ratio: 0.3
 # Genesis
@@ -21,6 +22,8 @@ geth_metrics_port: '{{ 6060 + idx|int + 1 }}'
 geth_rpc_enabled: true
 geth_rpc_addr: '0.0.0.0'
 geth_rpc_port: '{{ 8545 + idx|int + 1 }}'
+geth_authrpc_addr: '0.0.0.0'
+geth_authrpc_port: '{{ 8551 + idx|int + 1 }}'
 geth_websocket_enabled: true
 geth_websocket_addr: '0.0.0.0'
 geth_websocket_port: '{{ 9546 + idx|int + 1 }}'
@@ -35,7 +38,7 @@ beacon_node_build_frequency: 'daily'
 beacon_node_validator_monitor_auto: true
 beacon_node_validator_monitor_totals: true
 # Eth1 Sync
-beacon_node_web3_urls: ['ws://{{ hostname }}.wg:{{ geth_websocket_port }}']
+beacon_node_web3_urls: ['http://localhost:{{ geth_authrpc_port }}']
 # Validators from nimbus-private repo
 beacon_node_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}'
 beacon_node_dist_validators_start: '{{ node.start | mandatory }}'

ansible/vars/nimbus-sepolia-metal.yml

@@ -4,8 +4,9 @@ geth_service_name: 'geth-sepolia-{{ "%02d"|format(idx|int+1) }}'
 geth_network_name: 'sepolia'
 geth_sync_mode: 'full'
 geth_log_level_name: info
-# Geth auth
+# Geth auth & JWT token
 geth_account_pass: '{{lookup("bitwarden", "nimbus/geth", field="password")}}'
+geth_authrpc_jwtsecret: '{{lookup("bitwarden", "nimbus/jwt-token")}}'
 # Memory settings
 geth_cont_mem_ratio: 0.3
 # Genesis
@@ -16,6 +17,8 @@ geth_metrics_port: '{{ 6060 + idx|int + 11 }}'
 geth_rpc_enabled: true
 geth_rpc_addr: '0.0.0.0'
 geth_rpc_port: '{{ 8545 + idx|int + 11 }}'
+geth_authrpc_addr: '0.0.0.0'
+geth_authrpc_port: '{{ 8551 + idx|int + 11 }}'
 geth_websocket_enabled: true
 geth_websocket_addr: '0.0.0.0'
 geth_websocket_port: '{{ 9546 + idx|int + 11 }}'
@@ -35,7 +38,7 @@ beacon_node_build_frequency: 'daily'
 beacon_node_validator_monitor_auto: true
 beacon_node_validator_monitor_totals: true
 #Eth1 Sync
-beacon_node_web3_urls: ['ws://localhost:{{ geth_websocket_port }}']
+beacon_node_web3_urls: ['http://localhost:{{ geth_authrpc_port }}']
 # Validators from nimbus-private repo¬
 beacon_node_dist_validators_enabled: '{{ node.start is defined and node.end is defined }}'
 beacon_node_dist_validators_start: '{{ node.start | mandatory }}'