log-dash: configure Kibana dashboard

Signed-off-by: Jakub Sokołowski <jakub@status.im>

ansible/group_vars/all.yml

@@ -11,6 +11,9 @@ origin_certs:
     key: '{{lookup("passwordstore", "cloud/Cloudflare/status.im/origin.key returnall=true")}}'
     default: true
 
+# Consul Catalog Query URL
+consul_catalog_url: 'http://localhost:8500/v1/catalog/service'
+
 # Root password
 bootstrap__root_pass: '{{lookup("passwordstore", "hosts/admin-pass")}}'
 

ansible/group_vars/log-dash.yml

@@ -0,0 +1,56 @@
+---
+# Kibana Dashboard
+kibana_domain: 'nimbus-logs.status.im'
+kibana_version: '7.5.2'
+kibana_cont_name: kibana
+kibana_cont_port: 5601
+
+# Kibana LogTrail Plugin
+logtrail_kibana_version: '{{ kibana_version }}'
+logtrail_kibana_plugins_path: '{{ kibana_cont_vol }}/plugins'
+
+# oauth access
+oauth_domain: '{{ kibana_domain }}'
+oauth_cont_name: '{{ kibana_cont_name }}-oauth'
+oauth_upstream_cont: '{{ kibana_cont_name }}'
+oauth_upstream_port: '{{ kibana_cont_port }}'
+oauth_public_port: 443
+oauth_local_port: 8090
+oauth_cookie_secret: '{{ lookup("passwordstore", "services/cookie-secret") }}'
+oauth_id: '{{ lookup("passwordstore", "services/nimbus-kibana/oauth-id") }}'
+oauth_secret: '{{ lookup("passwordstore", "services/nimbus-kibana/oauth-secret") }}'
+
+# ElasticSearch Load Balancer
+es_lb_image: 'docker.elastic.co/elasticsearch/elasticsearch-oss:7.5.2'
+es_lb_cont_name: elasticsearch-lb
+es_lb_data_center: do-ams3
+es_lb_cluster_name: 'nimbus-logs-search'
+es_lb_api_port: 9200
+
+# JVM Memory settings
+es_lb_jvm_min_heap: 2g
+es_lb_jvm_max_heap: 2g
+
+# Nginx Proxy config
+nginx_sites:
+  kibana_http:
+    - listen 80
+    - server_name {{ oauth_domain }}
+    - return 302 https://$server_name$request_uri
+  kibana_ssl:
+    - listen 443 ssl
+
+    - server_name {{ oauth_domain }}
+
+    - ssl_certificate     /certs/origin.crt
+    - ssl_certificate_key /certs/origin.key
+
+    - location / {
+        proxy_set_header Host $host;
+        proxy_pass http://127.0.0.1:{{ oauth_local_port }}/;
+      }
+
+# Open Nginx Ports
+open_ports_protocol: tcp
+open_ports_comment: 'HTTP & HTTPS'
+open_ports_list: [80, 443]

ansible/logs.yml

@@ -14,3 +14,40 @@
       tags: nginx
     - role: open-ports
       tags: open-ports
+
+- name: Configure ElasticSearch servers
+  hosts: log-dash
+  roles:
+  pre_tasks:
+    - name: Fetch available elasticsearch nodes
+      uri:
+        url: '{{ consul_catalog_url }}/elasticsearch?dc={{ data_center }}&tag={{ es_lb_cluster_name }}'
+      register: es_services
+    - name: Extract ElasticSearch hostnames
+      set_fact:
+        # we also need to remove our own node from the list
+        es_lb_master_nodes: |
+          {{ es_services.json
+            | rejectattr("Node", "equalto", hostname)
+            | list
+            | json_query(
+            '[].{
+              name: Node,
+              addr: ServiceAddress,
+              port: to_string(ServiceMeta.cluster_port)
+            }') }}
+  roles:
+    - role: elasticsearch-lb
+      tags: elasticsearch-lb
+    - role: kibana
+      tags: kibana
+    - role: kibana-logtrail
+      tags: kibana-logtrail
+    - role: oauth-proxy
+      tags: oauth-proxy
+    - role: origin-certs
+      tags: origin-certs
+    - role: nginx
+      tags: nginx
+    - role: open-ports
+      tags: open-ports

ansible/requirements.yml

@@ -34,6 +34,14 @@
   src: git@github.com:status-im/infra-role-eth2-testnet-site.git
   scm: git
 
+- name: kibana
+  src: git@github.com:status-im/infra-role-kibana.git
+  scm: git
+
+- name: kibana-logtrail
+  src: git@github.com:status-im/infra-role-kibana-logtrail.git
+  scm: git
+
 - name: elasticsearch
   src: git@github.com:status-im/infra-role-elasticsearch.git
   scm: git
@@ -41,3 +49,7 @@
 - name: elasticsearch-hq
   src: git@github.com:status-im/infra-role-elasticsearch-hq.git
   scm: git
+
+- name: elasticsearch-lb
+  src: git@github.com:status-im/infra-role-elasticsearch-lb.git
+  scm: git