From 214c3d777d2a898774cc62ba8c185cfb7e9f8881 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 24 Jun 2021 20:17:33 -0300 Subject: [PATCH] Finished star-drive staging ansible --- ansible/README.md | 34 +++- .../configs/star-drive/flask/digitalocean.py | 65 ++++++++ ansible/configs/star-drive/nginx/default.conf | 27 +++ ansible/hosts.dev | 6 +- ansible/{ => scripts/star-drive}/runflask.sh | 2 +- ansible/star-drive.yml | 156 ++++++++++++------ 6 files changed, 230 insertions(+), 60 deletions(-) create mode 100644 ansible/configs/star-drive/flask/digitalocean.py create mode 100644 ansible/configs/star-drive/nginx/default.conf rename ansible/{ => scripts/star-drive}/runflask.sh (86%) diff --git a/ansible/README.md b/ansible/README.md index 8eb1cb9..4b3c415 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -1,7 +1,37 @@ #Ansible playbooks for server setups -digitalocean.yml contains a simple ansible playbook to set up a basic machine in Digital Ocean. +## Installing ansible on your machine -update the inventory with the correct IP and run it as follows: +You can find the instructions here: https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html + +These playbooks were tested on ansible 2.9.6 + + +## Basic server setup + +digitalocean.yml contains a simple ansible playbook to set up a basic ubuntu 20.04 machine in Digital Ocean. + +First create your droplet with the appropriate ssh keys. You could also do this with any VM you set up yourself + +Then update the inventory with the correct IP under `[digital_ocean]` and run it as follows: `ansible-playbook -i hosts.dev ./digitalocean.yml -u root` + +## Star Drive staging server setup + +`star-drive.yml` contains an ansible playbook to set up a star drive staging server. Please run the `digitalocean.yml` playbook against your server *first* + +Update the inventory with the correct IP under `[star-drive]` and run it as follows: + +`ansible-playbook -i hosts.dev ./star-drive.yml` + +Please note, this is a setup for staging purposes *ONLY*. It is not secure, API keys are not added, and while there is HTTP basic auth, there is no TLS (the password could be sniffed) + +As there is and only should be testing data on these servers, it should be ok. + +The database password is `ed_pass` by default, though it will prompt you for a password when you run the playbook. Should you wish to change this please update the configuration for flask in ./configs/star-drive/flask locally before running. If you specify another password on a subsequent run, it will be overwritten. + +The basic auth password is `pQNww!iXo9EEi988&kr` by default, though it will prompt you for a password when you run the playbook. Should you wish to change this please make a note of it. If you specify another password on a subsequent run, it will be overwritten. + + +There are a few things that could have been done more efficiently - however I wanted to demonstrate the use of ( and not ) of variables etc, so that people can learn. diff --git a/ansible/configs/star-drive/flask/digitalocean.py b/ansible/configs/star-drive/flask/digitalocean.py new file mode 100644 index 0000000..c5440a3 --- /dev/null +++ b/ansible/configs/star-drive/flask/digitalocean.py @@ -0,0 +1,65 @@ +import os +import re +from os import environ + +NAME = "STAR DRIVE Database" +VERSION = "0.1" + +CORS_ENABLED = True +CORS_ALLOW_ORIGINS = re.split(r',\s*', environ.get('CORS_ALLOW_ORIGINS', default="localhost:4200")) +DEVELOPMENT = True +TESTING = True + +MIRRORING = False +DELETE_RECORDS = True +EXPORT_CHECK_INTERNAL_MINUTES = 1 +IMPORT_INTERVAL_MINUTES = 1 + +SQLALCHEMY_DATABASE_URI = "postgresql://ed_user:ed_pass@localhost/stardrive" + + +# Elastic Search +ELASTIC_SEARCH = { + "index_prefix": "stardrive", + "hosts": ["localhost"], + "port": 9200, + "timeout": 20, + "verify_certs": False, + "use_ssl": False, + "http_auth_user": "", + "http_auth_pass": "" +} + +API_URL = "http://178.128.225.111:5000" +SITE_URL = "http://178.128.225.111" + +SECRET_KEY = 'stardrive_impossibly_bad_key_stored_in_public_repo_dont_use_this_outside_development_yuck!' + +FRONTEND_AUTH_CALLBACK = SITE_URL + "/#/session" +FRONTEND_EMAIL_RESET = SITE_URL + "/#/reset_password/" +FRONTEND_FORGOT_PASSWORD = SITE_URL + "/#/forgot-password" + +MAIL_SERVER = 'smtp.mailtrap.io' +MAIL_PORT = 2525 +MAIL_USE_SSL = False +MAIL_USE_TLS = True +MAIL_USERNAME = "YOUR-MAILTRAP-NAME - Copy these lines to your instance/config! edit there." +MAIL_PASSWORD = "YOUR-MAILTRAP-PASSWORD - Copy these lines to your instance/config! edit there." +MAIL_DEFAULT_SENDER='someaddress@fake.com' +MAIL_DEFAULT_USER='someaddress@fake.com' +MAIL_TIMEOUT = 10 + +GOOGLE_MAPS_API_KEY = "PRODUCTION_API_KEY_GOES_HERE" +GOOGLE_ANALYTICS_API_KEY = "PRODUCTION_API_KEY_GOES_HERE" + +ADMIN_EMAIL = "admin@tester.com" +PRINCIPAL_INVESTIGATOR_EMAIL = "pi@tester.com" # Receives some high levl alerts per agreement with InfoSec. + +ADMIN_PASSWORD_REGEX = r"((?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[\W]).{25})" +ADMIN_PASSWORD_INSTRUCTIONS = "Your password must be at least 25 characters long and contain at least one of each of " \ + "the following: uppercase letters, lowercase letters, numbers, and punctuation " \ + "characters. Don't use a password that you have used for any other purpose." +USER_PASSWORD_REGEX = r"((?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[\W]).{8})" +USER_PASSWORD_INSTRUCTIONS = "Your password must be at least 8 characters long, but we recommend 20 characters. It " \ + "should include at least one of each of the following: uppercase letters, " \ + "lowercase letters, numbers, and punctuation characters." diff --git a/ansible/configs/star-drive/nginx/default.conf b/ansible/configs/star-drive/nginx/default.conf new file mode 100644 index 0000000..23a17fe --- /dev/null +++ b/ansible/configs/star-drive/nginx/default.conf @@ -0,0 +1,27 @@ +server { + listen 80; + server_name localhost; + + access_log /var/log/nginx/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + auth_basic "Sartography Staging"; + auth_basic_user_file /etc/nginx/passwdfile; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /undefined/ { + if ($request_uri ~* "/undefined/(.*)") { + proxy_pass http://127.0.0.1:5000/$1; + } + auth_basic off; + } +} + + diff --git a/ansible/hosts.dev b/ansible/hosts.dev index 77ee4df..42f76e4 100644 --- a/ansible/hosts.dev +++ b/ansible/hosts.dev @@ -1,9 +1,9 @@ [digital_ocean] -178.128.225.111 -138.197.144.144 +#138.197.144.144 +143.198.34.127 [MT] 138.197.144.144 [star-drive] -178.128.225.111 +143.198.34.127 diff --git a/ansible/runflask.sh b/ansible/scripts/star-drive/runflask.sh similarity index 86% rename from ansible/runflask.sh rename to ansible/scripts/star-drive/runflask.sh index db9395f..0b9c259 100644 --- a/ansible/runflask.sh +++ b/ansible/scripts/star-drive/runflask.sh @@ -1,8 +1,8 @@ #!/bin/bash export FLASK_APP=~/star-drive/backend/app/__init__.py source ~/python-venv/bin/activate +cd ~/star-drive/backend/ flask db migrate flask db upgrade flask initdb nohup flask run & - diff --git a/ansible/star-drive.yml b/ansible/star-drive.yml index 5411f80..9c028a4 100644 --- a/ansible/star-drive.yml +++ b/ansible/star-drive.yml @@ -4,14 +4,18 @@ db_name: stardrive vars_prompt: - name: "db_password" - prompt: "Please set a password for your database user" + prompt: "Please set a password for your database user" + default: "ed_pass" + - name: "ba_password" + prompt: "Please set a password for HTTP basic auth" + default: "pQNww!iXo9EEi988&kr" user: sartography gather_facts: no tasks: - name: get es key shell: wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - become: yes - + - name: add es repo ansible.builtin.apt_repository: repo: deb https://artifacts.elastic.co/packages/6.x/apt stable main @@ -19,7 +23,6 @@ filename: elastic-6.x become: yes - - name: Install JDK apt: name: openjdk-8-jdk @@ -41,6 +44,7 @@ - apt-transport-https - elasticsearch - apache2-dev + - python3-passlib become: yes - name: Install node with snap @@ -50,11 +54,48 @@ classic: yes channel: 9/stable become: yes - + + - name: Check whether swap is already enabled or not + shell: cat /etc/sysctl.conf + register: swap_enabled + become: true + + - block: + - name: create swap file + command: dd if=/dev/zero of=/swapfile bs=1G count=4 + + - name: change permission type + file: path=/swapfile mode=600 state=file + + - name: setup swap + command: mkswap /swapfile + + - name: create swap + command: swapon /swapfile + + - name: Add to fstab + action: lineinfile dest=/etc/fstab regexp="swapfile" line="/swapfile none swap sw 0 0" state=present + + - name: start swap + command: swapon -a + + - name: set swapiness + sysctl: + name: vm.swappiness + value: "10" + + - name: set swapiness + sysctl: + name: vm.vfs_cache_pressure + value: "50" + + when: swap_enabled.stdout.find('swappiness') == -1 + become: true + - name: "enable npm update checking" shell: "npm ; sudo chown -R $USER:$(id -gn $USER) /home/sartography/.config" - - name: Clone star-drive. + - name: Clone star-drive git: repo: https://github.com/sartography/star-drive.git version: staging @@ -69,6 +110,22 @@ virtualenv: ~/python-venv/ virtualenv_python: python3 + - name: run npm install + shell: npm install + + - name: Install Angular.js + npm: + name: "@angular/cli@10" + global: yes + state: present + become: yes + + - name: install angular prereqs + shell: cd ~/star-drive/frontend && npm install + + - name: compile angular server + shell: cd ~/star-drive/frontend && ng build + - name: bind to localhost for elastic search lineinfile: path: /etc/elasticsearch/elasticsearch.yml @@ -105,14 +162,6 @@ state: stopped become: yes - #- name: copy docker-compose file - # copy: - # src: ../../star-drive - # dest: /home/sartography - # mode: '600' - #- name: start docker-compose for star-drive dev - # shell: docker-compose -f /home/sartography/star-drive/docker-compose.yml up --detach - - name: Create db user, set password, grant privs community.postgresql.postgresql_user: name: "{{ db_user }}" @@ -171,16 +220,6 @@ become: true become_user: postgres - - name: run npm install - shell: npm install - - - name: Install Angular.js - npm: - name: "@angular/cli@10" - global: yes - state: present - become: yes - - name: Ensures {{project_root}}/conf dir exists file: path=~/star-drive/backend/instance state=directory - name: Copy configs @@ -195,49 +234,58 @@ src: ~/star-drive/backend/instance/config/default.py dest: ~/star-drive/backend/instance/config.py remote_src: yes - + - name: Copy init script copy: - src: ./runflask.sh + src: ./scripts/star-drive/runflask.sh dest: ~/ + - name: Copy flask config into the correct location + copy: + src: ./configs/star-drive/flask/digitalocean.py + dest: ~/star-drive/backend/instance/config.py + + - name: Copy nginx config into the correct location + copy: + src: ./configs/star-drive/nginx/default.conf + dest: /etc/nginx/conf.d/default.conf + become: yes + - name: run flask server shell: bash ~/runflask.sh environment: FLASK_APP: ~/star-drive/app/__init__.py - - name: install angular prereqs - shell: cd ~/star-drive/frontend && npm install - - #- name: compile angular server - # shell: cd ~/star-drive/frontend && ng build - - - name: copy dists to www-data + - name: copy dists to html dir copy: - src: /home/elad/sartography/star-drive/frontend/dist/star-drive/ - dest: /var/www/ - #remote_src: yes - owner: www-data - group: www-data + src: /home/sartography/star-drive/frontend/dist/star-drive/ + dest: /usr/share/nginx/html/ + remote_src: yes + owner: nginx + group: nginx mode: '0644' become: yes + - name: fix directory permissions for nginx + command: find /usr/share/nginx/html/ -type d -exec chmod -c 0755 {} \; + register: chmod_result + changed_when: "chmod_result.stdout != \"\"" + become: yes + - name: Add a user to a password file and ensure permissions are set + community.general.htpasswd: + path: /etc/nginx/passwdfile + name: sartography + password: "{{ ba_password }}" + owner: root + group: nginx + mode: 0640 + become: yes - #- name: migrate flask db - # shell: bash "source ~/python-venv/bin/activate && flask db migrate" - # environment: - # FLASK_APP: ~/star-dive/app/__init__.py - - #- name: init flask db - # shell: flask initdb - # environment: - # FLASK_APP: ~/star-dive/app/__init__.py - - #- name: run flask server - # shell: nohup flask run & - # environment: - # FLASK_APP: ~/star-dive/app/__init__.py - - - + - name: restart and enable service pick up config changes + systemd: + state: restarted + enabled: yes + name: nginx + become: yes +