add store nodes and configure nim-waku

ansible/group_vars/boot.yml

@@ -1,5 +1,4 @@
 ---
-# Custom SSH accounts for Shards fleet, should start from UID 8000.
 
 # Tag dependent on fleet: test
 nim_waku_cont_tag: 'deploy-shards-{{ stage }}'

ansible/group_vars/store.yml

@@ -0,0 +1,59 @@
+---
+
+# Tag dependent on fleet: test
+nim_waku_cont_tag: 'deploy-shards-{{ stage }}'
+nim_waku_cont_name: 'nim-waku-store'
+# TODO: switch for traces foe 1 node in host_vars
+nim_waku_log_level: 'debug'
+nim_waku_protocols_enabled: ['relay', 'store']
+nim_waku_disc_v5_enabled: true
+nim_waku_dns4_domain_name: '{{ dns_entry }}'
+nim_waku_node_key: '{{lookup("bitwarden", "fleets/shards/"+stage+"/nodekeys", field=hostname)}}'
+
+# Topic configuration
+nim_waku_pubsub_topics:
+  - '/waku/2/rs/16/128'
+  - '/waku/2/rs/16/256'
+
+nim_waku_protected_topics:
+  - '/waku/2/rs/16/128:045ced3b90fabf7673c5165f9cc3a038fd2cfeb96946538089c310b5eaa3a611094b27d8216d9ec8110bd0e0e9fa7a7b5a66e86a27954c9d88ebd41d0ab6cfbb91'
+  - '/waku/2/rs/16/256:049022b33f7583f34463f5b7622e5da29f99f993e6858a478465c68ee114ccf142204eff285ed922349c4b71b178a2e1a2154b99bcc2d8e91b3994626ffa9f1a6c'
+
+# Ports
+nim_waku_p2p_tcp_port: 30303
+nim_waku_p2p_udp_port: 30303
+nim_waku_metrics_port: 8008
+nim_waku_disc_v5_port: 9000
+nim_waku_rpc_tcp_port: 8545
+
+# Limits
+nim_waku_p2p_max_connections: 300
+
+# Store
+nim_waku_store_message_retention_policy: 'time:2592000' # 30 days
+
+# DNS Discovery
+# TODO: enable DNS discovery
+nim_waku_dns_disc_enabled: false
+#nim_waku_dns_disc_url: 'enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@{{ stage }}.nodes.status.im'
+
+# Enable WebSockets via Websockify
+nim_waku_websockify_enabled: false
+
+# Enable websockets in Waku
+nim_waku_websocket_enabled: false
+
+# Consul Service
+nim_waku_consul_success_before_passing: 5
+nim_waku_consul_failures_before_warning: 2
+nim_waku_consul_failures_before_critical: 20
+
+# Open LibP2P Ports
+open_ports_default_comment: '{{ nim_waku_cont_name }}'
+open_ports_default_chain: 'SERVICES'
+open_ports_default_protocol: 'tcp'
+open_ports_list:
+  - { port: '{{ nim_waku_p2p_udp_port }}', protocol: 'udp' }
+  - { port: '{{ nim_waku_p2p_tcp_port }}' }
+  - { port: '{{ nim_waku_disc_v5_port }}', protocol: 'udp' }
+  - { port: '{{ nim_waku_metrics_port }}', chain: 'VPN', ipset: 'metrics.hq' }

ansible/inventory/test

@@ -7,10 +7,18 @@ boot-01.gc-us-central1-a.shards.test hostname=boot-01.gc-us-central1-a.shards.te
 boot-02.ac-cn-hongkong-c.shards.test hostname=boot-02.ac-cn-hongkong-c.shards.test ansible_host=8.218.174.108 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=boot-02.ac-cn-hongkong-c.shards.test.statusim.net
 boot-02.do-ams3.shards.test hostname=boot-02.do-ams3.shards.test ansible_host=178.128.143.241 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=boot-02.do-ams3.shards.test.statusim.net
 boot-02.gc-us-central1-a.shards.test hostname=boot-02.gc-us-central1-a.shards.test ansible_host=34.31.14.239 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=boot-02.gc-us-central1-a.shards.test.statusim.net
+store-01.ac-cn-hongkong-c.shards.test hostname=store-01.ac-cn-hongkong-c.shards.test ansible_host=8.218.74.73 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=store-01.ac-cn-hongkong-c.shards.test.statusim.net
+store-01.do-ams3.shards.test hostname=store-01.do-ams3.shards.test ansible_host=159.223.242.94 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=store-01.do-ams3.shards.test.statusim.net
+store-01.gc-us-central1-a.shards.test hostname=store-01.gc-us-central1-a.shards.test ansible_host=34.170.192.39 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=store-01.gc-us-central1-a.shards.test.statusim.net
+store-02.ac-cn-hongkong-c.shards.test hostname=store-02.ac-cn-hongkong-c.shards.test ansible_host=8.218.121.232 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=store-02.ac-cn-hongkong-c.shards.test.statusim.net
+store-02.do-ams3.shards.test hostname=store-02.do-ams3.shards.test ansible_host=159.223.242.154 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=store-02.do-ams3.shards.test.statusim.net
+store-02.gc-us-central1-a.shards.test hostname=store-02.gc-us-central1-a.shards.test ansible_host=34.170.154.2 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=store-02.gc-us-central1-a.shards.test.statusim.net
 
 [ac-cn-hongkong-c]
 boot-01.ac-cn-hongkong-c.shards.test
 boot-02.ac-cn-hongkong-c.shards.test
+store-01.ac-cn-hongkong-c.shards.test
+store-02.ac-cn-hongkong-c.shards.test
 
 [boot]
 boot-01.ac-cn-hongkong-c.shards.test
@@ -23,10 +31,14 @@ boot-02.gc-us-central1-a.shards.test
 [do-ams3]
 boot-01.do-ams3.shards.test
 boot-02.do-ams3.shards.test
+store-01.do-ams3.shards.test
+store-02.do-ams3.shards.test
 
 [gc-us-central1-a]
 boot-01.gc-us-central1-a.shards.test
 boot-02.gc-us-central1-a.shards.test
+store-01.gc-us-central1-a.shards.test
+store-02.gc-us-central1-a.shards.test
 
 [shards.test]
 boot-01.ac-cn-hongkong-c.shards.test
@@ -35,4 +47,18 @@ boot-01.gc-us-central1-a.shards.test
 boot-02.ac-cn-hongkong-c.shards.test
 boot-02.do-ams3.shards.test
 boot-02.gc-us-central1-a.shards.test
+store-01.ac-cn-hongkong-c.shards.test
+store-01.do-ams3.shards.test
+store-01.gc-us-central1-a.shards.test
+store-02.ac-cn-hongkong-c.shards.test
+store-02.do-ams3.shards.test
+store-02.gc-us-central1-a.shards.test
+
+[store]
+store-01.ac-cn-hongkong-c.shards.test
+store-01.do-ams3.shards.test
+store-01.gc-us-central1-a.shards.test
+store-02.ac-cn-hongkong-c.shards.test
+store-02.do-ams3.shards.test
+store-02.gc-us-central1-a.shards.test
 

ansible/main.yml

@@ -9,9 +9,16 @@
     - local_action: command ./versioncheck.py
       changed_when: false
 
-- name: Configure Waku Nodes
+- name: Configure Waku Bootstrap Nodes
   hosts: boot
   roles:
         - { role: open-ports,       tags: open-ports }
         - { role: swap-file,        tags: swap-file  }
         - { role: nim-waku,         tags: nim-waku   }
+
+- name: Configure Waku Storage Nodes
+  hosts: store
+  roles:
+        - { role: open-ports,       tags: open-ports }
+        - { role: swap-file,        tags: swap-file  }
+        - { role: nim-waku,         tags: nim-waku   }

ansible/requirements.yml

@@ -17,12 +17,10 @@
 
 - name: consul-service
   src: git@github.com:status-im/infra-role-consul-service.git
-  version: 4d7c9c606f5e039a22c34ba93961d05056c0e002
   scm: git
 
 - name: systemd-timer
   src: git@github.com:status-im/infra-role-systemd-timer.git
-  version: 24b9f3c82b0f2cc89211e40b0abce07e983e67c1
   scm: git
 
 - name: nim-waku

hosts_store.tf

@@ -0,0 +1,29 @@
+module "store" {
+  source = "github.com/status-im/infra-tf-multi-provider"
+
+  /* node type */
+  name   = "store"
+  group  = "store"
+  env    = "shards"
+  stage  = terraform.workspace
+  domain = var.domain
+
+  /* scaling */
+  host_count = local.ws["store_hosts_count"]
+
+  /* instance sizes */
+  do_type = local.ws["do_type"] /* DigitalOcean */
+  ac_type = local.ws["ac_type"] /* Alibaba Cloud */
+  gc_type = local.ws["gc_type"] /* Google Cloud */
+
+  /* data volumes */
+  ac_data_vol_size = local.ws["data_vol_size"]
+  do_data_vol_size = local.ws["data_vol_size"]
+  gc_data_vol_size = local.ws["data_vol_size"]
+
+  /* firewall */
+  open_udp_ports = [
+    "9000", /* discovery v5 */
+  ]
+}
+

workspaces.tf

@@ -12,6 +12,7 @@ locals {
       /* Default settings for all fleets/workspaces. */
 
       boot_hosts_count = 2
+      store_hosts_count = 2
 
       do_type = "s-1vcpu-2gb" /* DigitalOcean */
       ac_type = "ecs.t5-lc1m2.small" /* Alibaba Cloud */