switch to multi-DB layout with one DB per DC
Based on research done in this issue the latency introduced by cross-DC database operations is too high to be vaiable: https://github.com/status-im/infra-status/issues/37 Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
parent
74c0fab223
commit
02b75ec688
|
@ -23,7 +23,7 @@ nim_waku_rpc_tcp_addr: 0.0.0.0
|
||||||
# Limits
|
# Limits
|
||||||
nim_waku_p2p_max_connections: 200
|
nim_waku_p2p_max_connections: 200
|
||||||
# Store
|
# Store
|
||||||
nim_waku_store_message_db_host: 'db-01.do-ams3.status.test.wg'
|
nim_waku_store_message_db_host: 'db-01.{{ data_center }}.status.test.wg'
|
||||||
nim_waku_store_message_db_name: 'nim-waku'
|
nim_waku_store_message_db_name: 'nim-waku'
|
||||||
nim_waku_store_message_db_user: 'nim-waku'
|
nim_waku_store_message_db_user: 'nim-waku'
|
||||||
nim_waku_store_message_db_pass: '{{lookup("bitwarden", "fleets/status/"+stage+"/db/nim-waku")}}'
|
nim_waku_store_message_db_pass: '{{lookup("bitwarden", "fleets/status/"+stage+"/db/nim-waku")}}'
|
||||||
|
|
|
@ -1,12 +1,15 @@
|
||||||
# NOTE: This file is generated by terraform.py
|
# NOTE: This file is generated by terraform.py
|
||||||
# For emergency use when Consul fails
|
# For emergency use when Consul fails
|
||||||
[all]
|
[all]
|
||||||
db-01.do-ams3.status.test hostname=db-01.do-ams3.status.test ansible_host=178.128.136.200 env=status stage=test data_center=do-ams3 region=ams3 dns_entry=db-01.do-ams3.status.test.statusim.net
|
db-01.ac-cn-hongkong-c.status.test hostname=db-01.ac-cn-hongkong-c.status.test ansible_host=47.242.37.48 env=status stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=db-01.ac-cn-hongkong-c.status.test.statusim.net
|
||||||
|
db-01.do-ams3.status.test hostname=db-01.do-ams3.status.test ansible_host=206.189.240.151 env=status stage=test data_center=do-ams3 region=ams3 dns_entry=db-01.do-ams3.status.test.statusim.net
|
||||||
|
db-01.gc-us-central1-a.status.test hostname=db-01.gc-us-central1-a.status.test ansible_host=34.30.252.70 env=status stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=db-01.gc-us-central1-a.status.test.statusim.net
|
||||||
node-01.ac-cn-hongkong-c.status.test hostname=node-01.ac-cn-hongkong-c.status.test ansible_host=47.242.233.36 env=status stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=node-01.ac-cn-hongkong-c.status.test.statusim.net
|
node-01.ac-cn-hongkong-c.status.test hostname=node-01.ac-cn-hongkong-c.status.test ansible_host=47.242.233.36 env=status stage=test data_center=ac-cn-hongkong-c region=cn-hongkong-c dns_entry=node-01.ac-cn-hongkong-c.status.test.statusim.net
|
||||||
node-01.do-ams3.status.test hostname=node-01.do-ams3.status.test ansible_host=64.225.81.237 env=status stage=test data_center=do-ams3 region=ams3 dns_entry=node-01.do-ams3.status.test.statusim.net
|
node-01.do-ams3.status.test hostname=node-01.do-ams3.status.test ansible_host=64.225.81.237 env=status stage=test data_center=do-ams3 region=ams3 dns_entry=node-01.do-ams3.status.test.statusim.net
|
||||||
node-01.gc-us-central1-a.status.test hostname=node-01.gc-us-central1-a.status.test ansible_host=34.122.252.118 env=status stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=node-01.gc-us-central1-a.status.test.statusim.net
|
node-01.gc-us-central1-a.status.test hostname=node-01.gc-us-central1-a.status.test ansible_host=34.122.252.118 env=status stage=test data_center=gc-us-central1-a region=us-central1-a dns_entry=node-01.gc-us-central1-a.status.test.statusim.net
|
||||||
|
|
||||||
[ac-cn-hongkong-c]
|
[ac-cn-hongkong-c]
|
||||||
|
db-01.ac-cn-hongkong-c.status.test
|
||||||
node-01.ac-cn-hongkong-c.status.test
|
node-01.ac-cn-hongkong-c.status.test
|
||||||
|
|
||||||
[do-ams3]
|
[do-ams3]
|
||||||
|
@ -14,10 +17,13 @@ db-01.do-ams3.status.test
|
||||||
node-01.do-ams3.status.test
|
node-01.do-ams3.status.test
|
||||||
|
|
||||||
[gc-us-central1-a]
|
[gc-us-central1-a]
|
||||||
|
db-01.gc-us-central1-a.status.test
|
||||||
node-01.gc-us-central1-a.status.test
|
node-01.gc-us-central1-a.status.test
|
||||||
|
|
||||||
[status-db]
|
[status-db]
|
||||||
|
db-01.ac-cn-hongkong-c.status.test
|
||||||
db-01.do-ams3.status.test
|
db-01.do-ams3.status.test
|
||||||
|
db-01.gc-us-central1-a.status.test
|
||||||
|
|
||||||
[status-node]
|
[status-node]
|
||||||
node-01.ac-cn-hongkong-c.status.test
|
node-01.ac-cn-hongkong-c.status.test
|
||||||
|
@ -25,7 +31,9 @@ node-01.do-ams3.status.test
|
||||||
node-01.gc-us-central1-a.status.test
|
node-01.gc-us-central1-a.status.test
|
||||||
|
|
||||||
[status.test]
|
[status.test]
|
||||||
|
db-01.ac-cn-hongkong-c.status.test
|
||||||
db-01.do-ams3.status.test
|
db-01.do-ams3.status.test
|
||||||
|
db-01.gc-us-central1-a.status.test
|
||||||
node-01.ac-cn-hongkong-c.status.test
|
node-01.ac-cn-hongkong-c.status.test
|
||||||
node-01.do-ams3.status.test
|
node-01.do-ams3.status.test
|
||||||
node-01.gc-us-central1-a.status.test
|
node-01.gc-us-central1-a.status.test
|
||||||
|
|
|
@ -9,6 +9,14 @@
|
||||||
- local_action: command ./versioncheck.py
|
- local_action: command ./versioncheck.py
|
||||||
changed_when: false
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Configure PostgreSQL DB
|
||||||
|
hosts: status-db
|
||||||
|
serial: '{{ serial|default(3) }}'
|
||||||
|
roles:
|
||||||
|
- { role: infra-role-open-ports, tags: open-ports }
|
||||||
|
- { role: infra-role-swap-file, tags: swap-file }
|
||||||
|
- { role: infra-role-postgres-ha, tags: postgres-ha }
|
||||||
|
|
||||||
- name: Configure Waku Nodes
|
- name: Configure Waku Nodes
|
||||||
hosts: status-node
|
hosts: status-node
|
||||||
serial: '{{ serial|default(3) }}'
|
serial: '{{ serial|default(3) }}'
|
||||||
|
@ -17,11 +25,3 @@
|
||||||
- { role: infra-role-swap-file, tags: swap-file }
|
- { role: infra-role-swap-file, tags: swap-file }
|
||||||
- { role: infra-role-certbot, tags: certbot }
|
- { role: infra-role-certbot, tags: certbot }
|
||||||
- { role: infra-role-nim-waku, tags: nim-waku }
|
- { role: infra-role-nim-waku, tags: nim-waku }
|
||||||
|
|
||||||
- name: Configure PostgreSQL DB
|
|
||||||
hosts: status-db
|
|
||||||
serial: '{{ serial|default(3) }}'
|
|
||||||
roles:
|
|
||||||
- { role: infra-role-open-ports, tags: open-ports }
|
|
||||||
- { role: infra-role-swap-file, tags: swap-file }
|
|
||||||
- { role: infra-role-postgres-ha, tags: postgres-ha }
|
|
||||||
|
|
15
db.tf
15
db.tf
|
@ -1,5 +1,5 @@
|
||||||
module "db" {
|
module "db" {
|
||||||
source = "github.com/status-im/infra-tf-digital-ocean"
|
source = "github.com/status-im/infra-tf-multi-provider"
|
||||||
|
|
||||||
/* node type */
|
/* node type */
|
||||||
name = "db"
|
name = "db"
|
||||||
|
@ -9,9 +9,18 @@ module "db" {
|
||||||
domain = var.domain
|
domain = var.domain
|
||||||
|
|
||||||
/* scaling */
|
/* scaling */
|
||||||
type = local.ws["db_do_type"]
|
|
||||||
host_count = local.ws["db_count"]
|
host_count = local.ws["db_count"]
|
||||||
|
|
||||||
|
/* instance type */
|
||||||
|
do_type = local.ws["db_do_type"]
|
||||||
|
ac_type = local.ws["db_ac_type"]
|
||||||
|
gc_type = local.ws["db_gc_type"]
|
||||||
|
|
||||||
/* data volumes */
|
/* data volumes */
|
||||||
data_vol_size = local.ws["db_data_vol_size"]
|
do_data_vol_size = local.ws["db_data_vol_size"]
|
||||||
|
ac_root_vol_size = local.ws["db_data_vol_size"]
|
||||||
|
gc_root_vol_size = local.ws["db_data_vol_size"]
|
||||||
|
|
||||||
|
/* fix for volume size */
|
||||||
|
ac_data_vol_type = local.ws["db_ac_data_vol_type"]
|
||||||
}
|
}
|
||||||
|
|
8
node.tf
8
node.tf
|
@ -15,14 +15,6 @@ module "node" {
|
||||||
ac_type = local.ws["node_ac_type"] /* Alibaba Cloud */
|
ac_type = local.ws["node_ac_type"] /* Alibaba Cloud */
|
||||||
gc_type = local.ws["node_gc_type"] /* Google Cloud */
|
gc_type = local.ws["node_gc_type"] /* Google Cloud */
|
||||||
|
|
||||||
/* data volumes */
|
|
||||||
ac_data_vol_size = local.ws["node_data_vol_size"]
|
|
||||||
do_data_vol_size = local.ws["node_data_vol_size"]
|
|
||||||
gc_data_vol_size = local.ws["node_data_vol_size"]
|
|
||||||
|
|
||||||
/* fix for volume size */
|
|
||||||
ac_data_vol_type = local.ws["node_ac_data_vol_type"]
|
|
||||||
|
|
||||||
/* firewall */
|
/* firewall */
|
||||||
open_tcp_ports = [
|
open_tcp_ports = [
|
||||||
"30303", /* p2p main */
|
"30303", /* p2p main */
|
||||||
|
|
|
@ -16,24 +16,25 @@ locals {
|
||||||
/* Waku nodes */
|
/* Waku nodes */
|
||||||
node_count = 1
|
node_count = 1
|
||||||
node_do_type = "s-1vcpu-2gb" /* DigitalOcean */
|
node_do_type = "s-1vcpu-2gb" /* DigitalOcean */
|
||||||
node_ac_type = "ecs.t5-lc1m2.small" /* Alibaba Cloud */
|
node_ac_type = "ecs.t5-lc1m2.small" /* AlibabaCloud */
|
||||||
node_gc_type = "g1-small" /* Google Cloud */
|
node_gc_type = "g1-small" /* GoogleCloud */
|
||||||
node_ac_data_vol_type = "cloud_ssd"
|
|
||||||
node_data_vol_size = 40 /* TODO Remove in favor of DB hosts. */
|
|
||||||
|
|
||||||
/* PostgreSQL */
|
/* PostgreSQL */
|
||||||
db_count = 1
|
db_count = 1
|
||||||
db_do_type = "s-1vcpu-2gb" /* DigitalOcean */
|
db_do_type = "s-1vcpu-2gb" /* DigitalOcean */
|
||||||
|
db_ac_type = "ecs.t5-lc1m2.small" /* AlibabaCloud */
|
||||||
|
db_gc_type = "g1-small" /* GoogleCloud */
|
||||||
|
db_ac_data_vol_type = "cloud_ssd"
|
||||||
db_data_vol_size = 40
|
db_data_vol_size = 40
|
||||||
}
|
}
|
||||||
|
|
||||||
# Inherits defaults.
|
# Inherits defaults.
|
||||||
test = {
|
test = {}
|
||||||
db_do_type = "s-6vcpu-16gb"
|
|
||||||
}
|
|
||||||
prod = {
|
prod = {
|
||||||
|
db_do_type = "s-2vcpu-4gb"
|
||||||
|
db_ac_type = "ecs.sn1ne.large"
|
||||||
|
db_do_type = "c2d-highcpu-2"
|
||||||
node_count = 2
|
node_count = 2
|
||||||
node_data_vol_size = 350
|
|
||||||
db_data_vol_size = 300
|
db_data_vol_size = 300
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue