deploy elasticsearch and a kibana dashboard

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/group_vars/log-dash.yml

@@ -0,0 +1,60 @@
+---
+# Kibana Dashboard
+kibana_domain: 'nimbus-logs.infra.status.im'
+kibana_version: '7.8.0'
+kibana_cont_name: kibana
+kibana_cont_port: 5601
+
+# Kibana LogTrail Plugin
+logtrail_kibana_version: '{{ kibana_version }}'
+logtrail_kibana_plugins_path: '{{ kibana_cont_vol }}/plugins'
+
+# oauth access
+oauth_domain: '{{ kibana_domain }}'
+oauth_cont_name: '{{ kibana_cont_name }}-oauth'
+oauth_upstream_cont: '{{ kibana_cont_name }}'
+oauth_upstream_port: '{{ kibana_cont_port }}'
+oauth_public_port: 443
+oauth_local_port: 8090
+oauth_cookie_secret: '{{ lookup("passwordstore", "services/cookie-secret") }}'
+oauth_id: '{{ lookup("passwordstore", "services/nimbus-kibana/oauth-id") }}'
+oauth_secret: '{{ lookup("passwordstore", "services/nimbus-kibana/oauth-secret") }}'
+
+# ElasticSearch Load Balancer
+es_lb_image: 'docker.elastic.co/elasticsearch/elasticsearch-oss:7.8.0'
+es_lb_cont_name: elasticsearch-lb
+es_lb_data_center: do-ams3
+es_lb_cluster_name: 'nimbus-logs-search'
+es_lb_api_port: 9200
+
+logclean_es_host: 'localhost'
+logclean_es_port: '{{ es_lb_api_port }}'
+
+# JVM Memory settings
+es_lb_jvm_min_heap: 2g
+es_lb_jvm_max_heap: 2g
+
+# Nginx Proxy config
+nginx_sites:
+  kibana_http:
+    - listen 80
+    - server_name {{ oauth_domain }}
+    - return 302 https://$server_name$request_uri
+  kibana_ssl:
+    - listen 443 ssl
+
+    - server_name {{ oauth_domain }}
+
+    - ssl_certificate     /certs/origin.crt
+    - ssl_certificate_key /certs/origin.key
+
+    - location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:{{ oauth_local_port }}/;
+      }
+
+# Open Nginx Ports
+open_ports_default_comment: 'HTTP & HTTPS'
+open_ports_list:
+  - { port: 80,  protocol: 'tcp' }
+  - { port: 443, protocol: 'tcp' }

ansible/group_vars/log-store.yml

@@ -0,0 +1,72 @@
+---
+# Mount extra volume for ElasticSearch container
+bootstrap__extra_volume_path: '/docker'
+
+# ElasticSearch Cluster
+es_master_nodes: |
+  {{ ansible_play_hosts
+   | map('extract', hostvars)
+   | list
+   | json_query(
+   '[].{
+     name: hostname,
+     addr: ansible_local.tinc.vpn_ip,
+     port: to_string(es_node_port)
+   }') }}
+es_cluster_name: 'nimbus-logs-search'
+es_cont_name: 'elasticsearch'
+
+es_image: 'docker.elastic.co/elasticsearch/elasticsearch-oss:7.8.0'
+es_api_port: 9200
+es_node_port: 9300
+
+# Since Logstash stores 1 index per day this is fine
+# See: https://www.elastic.co/blog/how-many-shards-should-i-have-in-my-elasticsearch-cluster
+es_number_of_replicas: 1
+es_number_of_shards: 3
+
+# JVM memory settings
+es_jvm_mem_ratio: 0.6
+es_jvm_mem: '{{ (ansible_memtotal_mb * es_jvm_mem_ratio) | round(0, "floor") }}'
+es_jvm_min_heap: '{{ es_jvm_mem | int }}m'
+es_jvm_max_heap: '{{ es_jvm_mem | int }}m'
+
+# ElasticSearch HQ
+es_hq_lb_cont_name: '{{ es_cont_name }}'
+
+# oauth access
+oauth_domain: 'nimbus-es.infra.status.im'
+oauth_cont_name: '{{ es_hq_cont_name }}-oauth'
+oauth_upstream_cont: '{{ es_hq_cont_name }}'
+oauth_upstream_port: '{{ es_hq_cont_port }}'
+oauth_public_port: 443
+oauth_local_port: 8090
+oauth_cookie_secret: '{{ lookup("passwordstore", "services/cookie-secret") }}'
+oauth_id: '{{ lookup("passwordstore", "services/elastic-hq-nimbus/oauth-id") }}'
+oauth_secret: '{{ lookup("passwordstore", "services/elastic-hq-nimbus/oauth-secret") }}'
+
+# Proxy for ES HQ
+nginx_sites:
+  elasticsearch_hq_http:
+    - listen 80
+    - server_name {{ oauth_domain }}
+    - return 302 https://$server_name$request_uri
+  elasticsearch_hq_https:
+    - listen 80
+    - listen 443 ssl
+
+    - server_name {{ oauth_domain }}
+
+    - ssl_certificate     /certs/status.im/origin.crt
+    - ssl_certificate_key /certs/status.im/origin.key
+
+    - location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:{{ oauth_local_port }}/;
+      }
+
+# Open Nginx Ports
+open_ports_default_comment: 'HTTP & HTTPS'
+open_ports_list:
+  - { port: 80,  protocol: 'tcp' }
+  - { port: 443, protocol: 'tcp' }

ansible/requirements.yml

@@ -35,6 +35,26 @@
   src: git@github.com:status-im/infra-role-eth2-testnet-site.git
   scm: git
 
+- name: kibana
+  src: git@github.com:status-im/infra-role-kibana.git
+  scm: git
+
+- name: kibana-logtrail
+  src: git@github.com:status-im/infra-role-kibana-logtrail.git
+  scm: git
+
+- name: elasticsearch
+  src: git@github.com:status-im/infra-role-elasticsearch.git
+  scm: git
+
+- name: elasticsearch-hq
+  src: git@github.com:status-im/infra-role-elasticsearch-hq.git
+  scm: git
+
+- name: elasticsearch-lb
+  src: git@github.com:status-im/infra-role-elasticsearch-lb.git
+  scm: git
+
 - name: systemd-timer
   src: git@github.com:status-im/infra-role-systemd-timer.git
   scm: git