From 865908dc2f2dd49743d592edbcf42489ed5e79a5 Mon Sep 17 00:00:00 2001 From: Anton Iakimov Date: Fri, 8 Sep 2023 15:48:13 +0200 Subject: [PATCH] add store nodes and configure nim-waku --- ansible/group_vars/boot.yml | 1 - ansible/group_vars/store.yml | 59 ++++++++++++++++++++++++++++++++++++ ansible/inventory/test | 26 ++++++++++++++++ ansible/main.yml | 9 +++++- ansible/requirements.yml | 2 -- hosts_store.tf | 29 ++++++++++++++++++ workspaces.tf | 1 + 7 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 ansible/group_vars/store.yml create mode 100644 hosts_store.tf diff --git a/ansible/group_vars/boot.yml b/ansible/group_vars/boot.yml index f7a0d23..1eb1c71 100644 --- a/ansible/group_vars/boot.yml +++ b/ansible/group_vars/boot.yml @@ -1,5 +1,4 @@ --- -# Custom SSH accounts for Shards fleet, should start from UID 8000. # Tag dependent on fleet: test nim_waku_cont_tag: 'deploy-shards-{{ stage }}' diff --git a/ansible/group_vars/store.yml b/ansible/group_vars/store.yml new file mode 100644 index 0000000..63422b7 --- /dev/null +++ b/ansible/group_vars/store.yml @@ -0,0 +1,59 @@ +--- + +# Tag dependent on fleet: test +nim_waku_cont_tag: 'deploy-shards-{{ stage }}' +nim_waku_cont_name: 'nim-waku-store' +# TODO: switch for traces foe 1 node in host_vars +nim_waku_log_level: 'debug' +nim_waku_protocols_enabled: ['relay', 'store'] +nim_waku_disc_v5_enabled: true +nim_waku_dns4_domain_name: '{{ dns_entry }}' +nim_waku_node_key: '{{lookup("bitwarden", "fleets/shards/"+stage+"/nodekeys", field=hostname)}}' + +# Topic configuration +nim_waku_pubsub_topics: + - '/waku/2/rs/16/128' + - '/waku/2/rs/16/256' + +nim_waku_protected_topics: + - '/waku/2/rs/16/128:045ced3b90fabf7673c5165f9cc3a038fd2cfeb96946538089c310b5eaa3a611094b27d8216d9ec8110bd0e0e9fa7a7b5a66e86a27954c9d88ebd41d0ab6cfbb91' + - '/waku/2/rs/16/256:049022b33f7583f34463f5b7622e5da29f99f993e6858a478465c68ee114ccf142204eff285ed922349c4b71b178a2e1a2154b99bcc2d8e91b3994626ffa9f1a6c' + +# Ports +nim_waku_p2p_tcp_port: 30303 +nim_waku_p2p_udp_port: 30303 +nim_waku_metrics_port: 8008 +nim_waku_disc_v5_port: 9000 +nim_waku_rpc_tcp_port: 8545 + +# Limits +nim_waku_p2p_max_connections: 300 + +# Store +nim_waku_store_message_retention_policy: 'time:2592000' # 30 days + +# DNS Discovery +# TODO: enable DNS discovery +nim_waku_dns_disc_enabled: false +#nim_waku_dns_disc_url: 'enrtree://AOGECG2SPND25EEFMAJ5WF3KSGJNSGV356DSTL2YVLLZWIV6SAYBM@{{ stage }}.nodes.status.im' + +# Enable WebSockets via Websockify +nim_waku_websockify_enabled: false + +# Enable websockets in Waku +nim_waku_websocket_enabled: false + +# Consul Service +nim_waku_consul_success_before_passing: 5 +nim_waku_consul_failures_before_warning: 2 +nim_waku_consul_failures_before_critical: 20 + +# Open LibP2P Ports +open_ports_default_comment: '{{ nim_waku_cont_name }}' +open_ports_default_chain: 'SERVICES' +open_ports_default_protocol: 'tcp' +open_ports_list: + - { port: '{{ nim_waku_p2p_udp_port }}', protocol: 'udp' } + - { port: '{{ nim_waku_p2p_tcp_port }}' } + - { port: '{{ nim_waku_disc_v5_port }}', protocol: 'udp' } + - { port: '{{ nim_waku_metrics_port }}', chain: 'VPN', ipset: 'metrics.hq' } diff --git a/ansible/inventory/test b/ansible/inventory/test index 73f8931..8a5b3f0 100644 --- a/ansible/inventory/test +++ b/ansible/inventory/test @@ -7,10 +7,18 @@ boot-01.gc-us-central1-a.shards.test hostname=boot-01.gc-us-central1-a.shards.te boot-02.ac-cn-hongkong-c.shards.test hostname=boot-02.ac-cn-hongkong-c.shards.test ansible_host=8.218.174.108 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=boot-02.ac-cn-hongkong-c.shards.test.statusim.net boot-02.do-ams3.shards.test hostname=boot-02.do-ams3.shards.test ansible_host=178.128.143.241 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=boot-02.do-ams3.shards.test.statusim.net boot-02.gc-us-central1-a.shards.test hostname=boot-02.gc-us-central1-a.shards.test ansible_host=34.31.14.239 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=boot-02.gc-us-central1-a.shards.test.statusim.net +store-01.ac-cn-hongkong-c.shards.test hostname=store-01.ac-cn-hongkong-c.shards.test ansible_host=8.218.74.73 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=store-01.ac-cn-hongkong-c.shards.test.statusim.net +store-01.do-ams3.shards.test hostname=store-01.do-ams3.shards.test ansible_host=159.223.242.94 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=store-01.do-ams3.shards.test.statusim.net +store-01.gc-us-central1-a.shards.test hostname=store-01.gc-us-central1-a.shards.test ansible_host=34.170.192.39 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=store-01.gc-us-central1-a.shards.test.statusim.net +store-02.ac-cn-hongkong-c.shards.test hostname=store-02.ac-cn-hongkong-c.shards.test ansible_host=8.218.121.232 env=shards stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=store-02.ac-cn-hongkong-c.shards.test.statusim.net +store-02.do-ams3.shards.test hostname=store-02.do-ams3.shards.test ansible_host=159.223.242.154 env=shards stage=test data_center=do-ams3 region=ams3 dns_entry=store-02.do-ams3.shards.test.statusim.net +store-02.gc-us-central1-a.shards.test hostname=store-02.gc-us-central1-a.shards.test ansible_host=34.170.154.2 env=shards stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=store-02.gc-us-central1-a.shards.test.statusim.net [ac-cn-hongkong-c] boot-01.ac-cn-hongkong-c.shards.test boot-02.ac-cn-hongkong-c.shards.test +store-01.ac-cn-hongkong-c.shards.test +store-02.ac-cn-hongkong-c.shards.test [boot] boot-01.ac-cn-hongkong-c.shards.test @@ -23,10 +31,14 @@ boot-02.gc-us-central1-a.shards.test [do-ams3] boot-01.do-ams3.shards.test boot-02.do-ams3.shards.test +store-01.do-ams3.shards.test +store-02.do-ams3.shards.test [gc-us-central1-a] boot-01.gc-us-central1-a.shards.test boot-02.gc-us-central1-a.shards.test +store-01.gc-us-central1-a.shards.test +store-02.gc-us-central1-a.shards.test [shards.test] boot-01.ac-cn-hongkong-c.shards.test @@ -35,4 +47,18 @@ boot-01.gc-us-central1-a.shards.test boot-02.ac-cn-hongkong-c.shards.test boot-02.do-ams3.shards.test boot-02.gc-us-central1-a.shards.test +store-01.ac-cn-hongkong-c.shards.test +store-01.do-ams3.shards.test +store-01.gc-us-central1-a.shards.test +store-02.ac-cn-hongkong-c.shards.test +store-02.do-ams3.shards.test +store-02.gc-us-central1-a.shards.test + +[store] +store-01.ac-cn-hongkong-c.shards.test +store-01.do-ams3.shards.test +store-01.gc-us-central1-a.shards.test +store-02.ac-cn-hongkong-c.shards.test +store-02.do-ams3.shards.test +store-02.gc-us-central1-a.shards.test diff --git a/ansible/main.yml b/ansible/main.yml index fa1c3e2..3503a1b 100644 --- a/ansible/main.yml +++ b/ansible/main.yml @@ -9,9 +9,16 @@ - local_action: command ./versioncheck.py changed_when: false -- name: Configure Waku Nodes +- name: Configure Waku Bootstrap Nodes hosts: boot roles: - { role: open-ports, tags: open-ports } - { role: swap-file, tags: swap-file } - { role: nim-waku, tags: nim-waku } + +- name: Configure Waku Storage Nodes + hosts: store + roles: + - { role: open-ports, tags: open-ports } + - { role: swap-file, tags: swap-file } + - { role: nim-waku, tags: nim-waku } diff --git a/ansible/requirements.yml b/ansible/requirements.yml index c0c75e5..731c071 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -17,12 +17,10 @@ - name: consul-service src: git@github.com:status-im/infra-role-consul-service.git - version: 4d7c9c606f5e039a22c34ba93961d05056c0e002 scm: git - name: systemd-timer src: git@github.com:status-im/infra-role-systemd-timer.git - version: 24b9f3c82b0f2cc89211e40b0abce07e983e67c1 scm: git - name: nim-waku diff --git a/hosts_store.tf b/hosts_store.tf new file mode 100644 index 0000000..4e8896e --- /dev/null +++ b/hosts_store.tf @@ -0,0 +1,29 @@ +module "store" { + source = "github.com/status-im/infra-tf-multi-provider" + + /* node type */ + name = "store" + group = "store" + env = "shards" + stage = terraform.workspace + domain = var.domain + + /* scaling */ + host_count = local.ws["store_hosts_count"] + + /* instance sizes */ + do_type = local.ws["do_type"] /* DigitalOcean */ + ac_type = local.ws["ac_type"] /* Alibaba Cloud */ + gc_type = local.ws["gc_type"] /* Google Cloud */ + + /* data volumes */ + ac_data_vol_size = local.ws["data_vol_size"] + do_data_vol_size = local.ws["data_vol_size"] + gc_data_vol_size = local.ws["data_vol_size"] + + /* firewall */ + open_udp_ports = [ + "9000", /* discovery v5 */ + ] +} + diff --git a/workspaces.tf b/workspaces.tf index 3371f9d..f5afc50 100644 --- a/workspaces.tf +++ b/workspaces.tf @@ -12,6 +12,7 @@ locals { /* Default settings for all fleets/workspaces. */ boot_hosts_count = 2 + store_hosts_count = 2 do_type = "s-1vcpu-2gb" /* DigitalOcean */ ac_type = "ecs.t5-lc1m2.small" /* Alibaba Cloud */