From a497da4dc83ac7eea18e8532038a947f99ef0223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Thu, 12 Sep 2024 15:32:45 +0200 Subject: [PATCH] add files from infra-office repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Sokołowski --- README.md | 61 +++++++++++++++++++++++++ USERS.md | 44 ++++++++++++++++++ defaults/main.yml | 59 ++++++++++++++++++++++++ handlers/main.yml | 6 +++ tasks/backup.yml | 19 ++++++++ tasks/consul.yml | 15 ++++++ tasks/container.yml | 33 ++++++++++++++ tasks/firewall.yml | 14 ++++++ tasks/main.yml | 6 +++ tasks/proxy.yml | 45 ++++++++++++++++++ templates/docker-compose.yml.j2 | 81 +++++++++++++++++++++++++++++++++ 11 files changed, 383 insertions(+) create mode 100644 README.md create mode 100644 USERS.md create mode 100644 defaults/main.yml create mode 100644 handlers/main.yml create mode 100644 tasks/backup.yml create mode 100644 tasks/consul.yml create mode 100644 tasks/container.yml create mode 100644 tasks/firewall.yml create mode 100644 tasks/main.yml create mode 100644 tasks/proxy.yml create mode 100644 templates/docker-compose.yml.j2 diff --git a/README.md b/README.md new file mode 100644 index 0000000..b47d157 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +# Description + +This role configures [HackMD](https://github.com/hackmdio/codimd), an open-source document editing platform for Status.im. + +# Configuration + +```yaml +hackmd_domain: 'notes.status.im' +# GitHub OAuth +hackmd_gh_oauth_id: 'super-secret-github-oauth-id' +hackmd_gh_oauth_secret: super-secret-github-oauth-key' +# Google OAuth +hackmd_gg_oauth_id: 'super-secret-google-oauth-id' +hackmd_gg_oauth_secret: super-secret-google-oauth-key' +``` + +# Management + +You can manage the containers using `docker-compose` command: +``` +admin@node-01.do-ams3.todo.misc:~ % cd /docker/hackmd +admin@node-01.do-ams3.todo.misc:/docker/hackmd % docker-compose --compatibility up --force-recreate -d +Recreating hackmd-db ... done +Recreating hackmd-app ... done +admin@node-01.do-ams3.todo.misc:/docker/hackmd % docker ps --filter=name=hackmd +CONTAINER ID NAMES IMAGE CREATED STATUS +15ebf1522b78 hackmd-app hackmdio/hackmd:2.3.2 3 seconds ago Up 1 second +fd7bf9523578 hackmd-db postgres:9.6-alpine 15 seconds ago Up 13 seconds +``` +For user management you can see the [`USERS.md`](./USERS.md) document. + +# Backups + +Backups are done via a [systemd timer](https://www.freedesktop.org/software/systemd/man/systemd.timer.html) and [`mongodump`](https://docs.mongodb.com/manual/reference/program/mongodump/): +``` + > sudo systemctl list-timers '*-hackmd-db.timer' +NEXT LEFT LAST PASSED UNIT ACTIVATES +Sat 2020-01-25 00:00:00 UTC 7h left n/a n/a backup-hackmd-db.timer backup-hackmd-db.service +Sat 2020-01-25 00:00:00 UTC 7h left n/a n/a dump-hackmd-db.timer dump-hackmd-db.service +``` +You can create an SQL backup of the PostgreSQL database by running: +``` + > sudo systemctl start dump-hackmd-db.service + > sudo systemctl status dump-hackmd-db.service +● dump-hackmd-db.service - Dump HackMD PostgreSQL database. + Loaded: loaded (/etc/systemd/system/dump-hackmd-db.service; static; vendor preset: enabled) + Active: inactive (dead) since Thu 2021-03-04 19:39:15 UTC; 7s ago +TriggeredBy: ● dump-hackmd-db.timer + Docs: https://github.com/status-im/infra-role-systemd-timer + Process: 867920 ExecStart=/usr/local/bin/dump-hackmd-db (code=exited, status=0/SUCCESS) + Main PID: 867920 (code=exited, status=0/SUCCESS) + +systemd[1]: Starting Dump HackMD PostgreSQL database.... +systemd[1]: dump-hackmd-db.service: Succeeded. +systemd[1]: Finished Dump HackMD PostgreSQL database.. +``` + +# Known Issues + +A bug in S3 library configuration makes S3 uploads unusable. +For more details see: https://github.com/hackmdio/codimd/issues/1572 diff --git a/USERS.md b/USERS.md new file mode 100644 index 0000000..8490afb --- /dev/null +++ b/USERS.md @@ -0,0 +1,44 @@ +# Description + +This file describes how users can be managed in HackMD. + +# Types + +There are three types of user antries in HackMD PostgreSQL database `Users` table. + +* Registered users - When registration is open these update `email` field. +* GitHub OAuth users - When logged in they fill `profile` with JSON from GH. +* Google OAuth users - When logged in they also fill `profile` + +We used to have open registrations, but those were closed. +Currently we only accept GitHub and Google logins. + +# Queries + +```sql +SELECT + u.id, + u.provider, + u.json->'username' AS username, + u.json->'displayName' AS name, + (CASE WHEN u.provider::text = '"github"' + THEN (u.json->'_json'->'email')::text + ELSE u.email END) AS email +FROM ( + SELECT + profileid AS id, + email AS email, + profile::json AS json, + profile::json->'provider' AS provider + FROM "Users") AS u; +``` +``` + id | provider | username | name | email +-----------------------+----------+--------------+---------------------------+------------------------ + 19521990 | "github" | "jlokier" | "Jamie Lokier" | "jamie@shareable.org" + 2212681 | "github" | "jakubgs" | "Jakub" | "jakub@gsokolowski.pl" + 116095778576207530385 | "google" | | "Jamie Lokier" | + 5702426 | "github" | "Samyoul" | "Samuel Hawksby-Robinson" | "samuel@samyoul.com" + 5483559 | "github" | "sachayves" | "Sacha Saint-Leger" | "sacha@ethereum.org" + ... +``` diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..233885f --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,59 @@ +--- +hackmd_domain: 'notes.example.org' +hackmd_service_name: 'hackmd' +hackmd_service_path: '/docker/{{ hackmd_service_name }}' +hackmd_service_compose: '{{ hackmd_service_path }}/docker-compose.yml' + +# Debug logs +hackmd_debug: false + +# App -------------------------------------------------------------------------- +hackmd_app_cont_name: '{{ hackmd_service_name }}-app' +hackmd_app_cont_tag: '2.4.2-1' +hackmd_app_cont_image: 'statusteam/hackmd:{{ hackmd_app_cont_tag }}' +hackmd_app_cont_port: 3001 +hackmd_app_cont_vol: '{{ hackmd_service_path }}/app' +hackmd_app_cont_uploads: '{{ hackmd_app_cont_vol }}/uploads' +hackmd_app_cont_uid: 1500 +hackmd_app_host_uid: '{{ 100000 + hackmd_app_cont_uid | int }}' +hackmd_session_life: '24h' +# hackmd_session_secret: 'changeMeIfYouCare' + +# DB --------------------------------------------------------------------------- +hackmd_db_cont_name: '{{ hackmd_service_name }}-db' +hackmd_db_cont_vol: '{{ hackmd_service_path }}/db' +hackmd_db_cont_image: 'postgres:9.6-alpine' +hackmd_db_cont_port: 5432 +hackmd_db_name: 'hackmd' +hackmd_db_user: 'hackmd' +hackmd_db_pass: 'changeMeIfYouCare' +hackmd_db_cont_uid: 70 +hackmd_db_host_uid: '{{ 100000 + hackmd_db_cont_uid | int }}' + +# Backups +hackmd_db_backup_service_name: 'dump-hackmd-db' +hackmd_db_backup_frequency: daily +hackmd_db_backup_timeout: 120 +hackmd_db_backup_user: root + +# s3 or digital ocean uploads +# WARNING: This is currently broken. +hackmd_s3_upload_enabled: false +hackmd_s3_access_key: ~ +hackmd_s3_secret_key: ~ +hackmd_s3_region: ~ +hackmd_s3_bucket: ~ +hackmd_s3_endpoint: ~ + +# google oauth +#hackmd_gg_oauth_id: ~ +#hackmd_gg_oauth_secret: ~ +# github oauth +#hackmd_gh_oauth_id: ~ +#hackmd_gh_oauth_secret: ~ +#hackmd_gh_oauth_orgs: [] + +# general container management +compose_recreate: 'smart' +compose_state: 'present' +compose_restart: false diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..ef626eb --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Save iptables rules + shell: iptables-save > /etc/iptables/rules.v4 + +- name: Restart nginx + service: name=nginx state=restarted diff --git a/tasks/backup.yml b/tasks/backup.yml new file mode 100644 index 0000000..5faf679 --- /dev/null +++ b/tasks/backup.yml @@ -0,0 +1,19 @@ +--- +- name: 'Create timer for MongoDB backup: {{ hackmd_db_backup_service_name }}' + include_role: name=infra-role-systemd-timer + vars: + systemd_timer_name: '{{ hackmd_db_backup_service_name }}' + systemd_timer_description: 'Dump HackMD PostgreSQL database.' + systemd_timer_user: '{{ hackmd_db_backup_user }}' + systemd_timer_frequency: '{{ hackmd_db_backup_frequency }}' + systemd_timer_timeout_sec: '{{ hackmd_db_backup_timeout }}' + systemd_timer_after_extra: 'docker.service' + systemd_timer_start_on_creation: false + systemd_timer_script_content: | + #!/usr/bin/env bash + BKP_DIR="{{ hackmd_db_cont_vol }}/backup/{{ hackmd_db_name }}" + rm -vfr "${BKP_DIR}" + /usr/bin/docker exec -i {{ hackmd_db_cont_name }} \ + pg_dump -F directory -f "/backup/{{ hackmd_db_name }}" \ + -U {{ hackmd_db_user }} {{ hackmd_db_name }} + chmod 750 -R "${BKP_DIR}" diff --git a/tasks/consul.yml b/tasks/consul.yml new file mode 100644 index 0000000..1a41ff1 --- /dev/null +++ b/tasks/consul.yml @@ -0,0 +1,15 @@ +--- +- name: HackMD | Create Consul service definition + include_role: name=infra-role-consul-service + vars: + consul_config_name: '{{ hackmd_service_name }}' + consul_services: + - name: '{{ hackmd_service_name }}' + tags: ['hackmd', 'notes', 'misc'] + # we advertise the port with basic auth + port: '{{ hackmd_app_cont_port }}' + checks: + - id: '{{ hackmd_service_name }}-status' + name: HackMD Healthcheck + type: http + http: 'http://localhost:{{ hackmd_app_cont_port }}/config' diff --git a/tasks/container.yml b/tasks/container.yml new file mode 100644 index 0000000..ffdfbf4 --- /dev/null +++ b/tasks/container.yml @@ -0,0 +1,33 @@ +--- +- name: HackMD | Create uploads directory file + file: + path: '{{ hackmd_app_cont_uploads }}' + state: directory + owner: '{{ hackmd_app_host_uid }}' + group: docker + recurse: true + +- name: HackMD | Create directory for DB data + file: + path: '{{ hackmd_db_cont_vol }}/data' + state: directory + owner: '{{ hackmd_db_host_uid }}' + group: dockremap + recurse: true + +- name: HackMD | Create compose file + template: + src: 'docker-compose.yml.j2' + dest: '{{ hackmd_service_compose }}' + owner: 'dockremap' + group: 'docker' + mode: 0640 + +- name: HackMD | Create containers + docker_compose: + project_src: '{{ hackmd_service_path }}' + pull: true + build: false + state: '{{ compose_state }}' + restarted: '{{ compose_restart }}' + recreate: '{{ compose_recreate | default("smart") }}' diff --git a/tasks/firewall.yml b/tasks/firewall.yml new file mode 100644 index 0000000..2c62ce3 --- /dev/null +++ b/tasks/firewall.yml @@ -0,0 +1,14 @@ +--- +- name: HackMD | Enable HTTP & HTTPS ports + iptables: + comment: '{{ hackmd_service_name }}' + chain: INPUT + jump: ACCEPT + source: '0.0.0.0/0' + protocol: 'tcp' + destination_port: '{{ item | int }}' + with_items: + - 80 + - 443 + notify: + - Save iptables rules diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..44ea225 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- import_tasks: container.yml +- import_tasks: consul.yml +- import_tasks: firewall.yml +- import_tasks: proxy.yml +- import_tasks: backup.yml diff --git a/tasks/proxy.yml b/tasks/proxy.yml new file mode 100644 index 0000000..cc1059a --- /dev/null +++ b/tasks/proxy.yml @@ -0,0 +1,45 @@ +--- +- name: HackMD | Configure nginx proxy + include_role: name=infra-role-nginx + vars: + nginx_configs: + hackmd_rate_limiting: + # Limit requests to 40 per minute based on IP + - limit_req_zone $binary_remote_addr zone=hackmd_by_ip:20m rate=40r/m + nginx_sites: + hackmd_http: + - listen 80 + - server_name {{ hackmd_domain }} + - return 301 https://{{ hackmd_domain }}$request_uri + + hackmd_ssl: + - listen 443 ssl + - server_name {{ hackmd_domain }} + + - ssl_certificate /certs/origin.crt + - ssl_certificate_key /certs/origin.key + + - proxy_set_header X-Frame-Options sameorigin + + # Script kiddies like scanning paths, so we rate limit most. + - location / { + limit_req zone=hackmd_by_ip burst=30 nodelay; + proxy_pass http://localhost:{{ hackmd_app_cont_port }}/; + proxy_set_header X-NginX-Proxy true; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_redirect off; + } + + # This path is used when editing so we can't rate limit it easily. + - location ~ ^/(socket.io|config|me) { + proxy_pass http://localhost:{{ hackmd_app_cont_port }}; + proxy_set_header X-NginX-Proxy true; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $host; + proxy_http_version 1.1; + proxy_redirect off; + } diff --git a/templates/docker-compose.yml.j2 b/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..ac455c7 --- /dev/null +++ b/templates/docker-compose.yml.j2 @@ -0,0 +1,81 @@ +--- +version: '3.7' +services: + node: + container_name: '{{ hackmd_app_cont_name }}' + image: '{{ hackmd_app_cont_image }}' + restart: always + ports: + - '{{ hackmd_app_cont_port }}:{{ hackmd_app_cont_port }}' + volumes: + - '{{ hackmd_app_cont_uploads }}:/home/hackmd/app/public/uploads' + environment: + # WARNING: HackMD expects lowercase string booleans! + DEBUG: '{{ hackmd_debug | string | lower }}' + CMD_LOGLEVEL: '{{ hackmd_debug | ternary("debug", "info") }}' + # Fixes syntax highlighting issues for some languages + CMD_USECDN: 'false' + CMD_PORT: '{{ hackmd_app_cont_port }}' + CMD_DB_URL: 'postgres://{{ hackmd_db_user }}:{{ hackmd_db_pass }}@db:{{ hackmd_db_cont_port }}/{{ hackmd_db_name }}' + CMD_DOMAIN: '{{ hackmd_domain }}' + CMD_ALLOW_ORIGIN: 'localhost, status.im' + CMD_URL_ADDPORT: 'false' + CMD_PROTOCOL_USESSL: 'true' + CMD_EMAIL: 'true' + CMD_ALLOW_EMAIL_REGISTER: 'false' + CMD_ALLOW_FREEURL: 'true' + CMD_ALLOW_PDF_EXPORT: 'true' + CMD_DEFAULT_PERMISSION: 'limited' + CMD_ALLOW_ANONYMOUS: 'false' + CMD_ALLOW_ANONYMOUS_EDITS: 'false' + CMD_ALLOW_ANONYMOUS_VIEWS: 'true' + CMD_GOOGLE_CLIENTID: '{{ hackmd_gg_oauth_id | mandatory }}' + CMD_GOOGLE_CLIENTSECRET: '{{ hackmd_gg_oauth_secret | mandatory }}' + CMD_GITHUB_CLIENTID: '{{ hackmd_gh_oauth_id | mandatory }}' + CMD_GITHUB_CLIENTSECRET: '{{ hackmd_gh_oauth_secret | mandatory }}' + CMD_GITHUB_ORGANIZATIONS: '{{ hackmd_gh_oauth_orgs | join(",") | mandatory }}' +{% if hackmd_s3_upload_enabled %} + # S3/DO Spaces uploads + CMD_IMAGE_UPLOAD_TYPE: 's3' + CMD_S3_ACCESS_KEY_ID: '{{ hackmd_s3_access_key }}' + CMD_S3_SECRET_ACCESS_KEY: '{{ hackmd_s3_secret_key }}' + CMD_S3_REGION: '{{ hackmd_s3_region }}' + CMD_S3_BUCKET: '{{ hackmd_s3_bucket }}' + CMD_S3_ENDPOINT: '{{ hackmd_s3_endpoint }}' +{% else %} + CMD_IMAGE_UPLOAD_TYPE: 'filesystem' +{% endif %} + CMD_SESSION_LIFE: '{{ hackmd_session_life | community.general.to_time_unit('ms') | int}}' +{% if hackmd_session_secret is defined and hackmd_session_secret %} + CMD_SESSION_SECRET: '{{ hackmd_session_secret }}' +{% endif %} + depends_on: + - 'db' + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:{{ hackmd_app_cont_port }}/config"] + interval: 30s + timeout: 10s + retries: 3 + + db: + container_name: '{{ hackmd_db_cont_name }}' + image: '{{ hackmd_db_cont_image }}' + restart: always + environment: + POSTGRES_DB: '{{ hackmd_db_name }}' + POSTGRES_USER: '{{ hackmd_db_user }}' + POSTGRES_PASSWORD: '{{ hackmd_db_pass }}' + # This fixes chmod errors on DB startup due to volume + userns-remap + PGDATA: '/var/lib/postgresql/data/pgdata' + ports: + - '{{ hackmd_db_cont_port }}:{{ hackmd_db_cont_port }}' + tmpfs: + - '/run/postgresql:size=512K' + - '/tmp:size=256K' + volumes: + - '{{ hackmd_db_cont_vol }}/data:/var/lib/postgresql/data' + - '{{ hackmd_db_cont_vol }}/backup:/backup' + healthcheck: + test: ["CMD", "pg_isready", "-U", "{{ hackmd_db_user }}"] + interval: 30s + retries: 5