diff --git a/.gitignore b/.gitignore index 470bdc1..f7a6069 100644 --- a/.gitignore +++ b/.gitignore @@ -182,9 +182,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder # .vscode/ @@ -214,4 +214,5 @@ __marimo__/ /data-dir /uploads *.DS_Store -*.pkl \ No newline at end of file +*.pkl +*.dockerignore diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2b1b737 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12-slim +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +# Folder is required only for status-backend and not the code +RUN rm -rf data-dir +# In case if the scripts are ran locally instead of with Docker +RUN rm -rf uploads +# Independent scripts +CMD ["sh", "-c", "python upload.py & python download.py"] diff --git a/README.md b/README.md index ff86b41..af33a65 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,12 @@ Monitoring tool for Status App communities ## Environment Variables -``` -POSTGRES_USERNAME -POSTGRES_PASSWORD -POSTGRES_DATABASE -POSTGRES_HOST -POSTGRES_PORT -``` +- `POSTGRES_USERNAME` - Postgres username. +- `POSTGRES_DATABASE` - The database name in the Postgres connection. +- `POSTGRES_HOST` - The Postgres host name that will be remotely connected to. +- `POSTGRES_PORT` - The Postgres port that will be remotely connected to. +- `STATUS_BACKEND_BASE_URL` (**OPTIONAL**) - The Status Backend URL. If you are running locally you do not need this variable (`localhost` will be automatically set). If you are running it in a Docker container, please set it to `status-backend` (as the `container_name` of the `docker-compose.yaml`). + ## Docker @@ -27,6 +26,7 @@ docker login harbor.status.im ```bash docker compose up ``` +**Note**: If you want to run Status Backend only, just keep the `status-backend` key in [`services`](./docker-compose.yaml). ## Python 1. Setup environment @@ -39,6 +39,7 @@ conda create -n status-monitoring python=3.12 ```bash pip install -r requirements.txt ``` + **Note**: If you are on Windows, you will have to install `psycopg2` instead of `psycopg2-binary`. ## Files @@ -47,4 +48,4 @@ pip install -r requirements.txt - `python create_account.py -u snt-maxxer -p StatusApp#123` - `python create_account.py --username snt-maxxer --password StatusApp#123` - `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`. -- `upload.py` - upload data from `download.py` to Postgres \ No newline at end of file +- `upload.py` - upload data from `download.py` to Postgres diff --git a/config.yaml b/config.yaml index b322502..60bdc51 100644 --- a/config.yaml +++ b/config.yaml @@ -1,19 +1,20 @@ postgres: - schema: "status_app_monitoring" - tables: - messages: "raw_messages" - community: "raw_community_info" - members: "raw_community_activity" + schema: "status_app_monitoring" + tables: + messages: "raw_messages" + community: "raw_community_info" + members: "raw_community_activity" status_app: - bot_name: "snt-maxxer" - channels: - - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW - - backend_params: - url: "http://localhost:8080" - logLevel: "INFO" - data_dir: "./data-dir" + bot_name: "snt-maxxer" + channels: + - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW + url: + port: 8080 + is_https: false + backend_params: + logLevel: "INFO" + data_dir: "./data-dir" sleep: # Values must be in MINUTES diff --git a/constants.py b/constants.py index 656cc5b..51e63c0 100644 --- a/constants.py +++ b/constants.py @@ -1,5 +1,6 @@ import os import yaml +from dotenv import load_dotenv as __load_dotenv CREDENTIALS_PATH = os.path.join(os.path.dirname(__file__), "accounts") UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads")) @@ -7,4 +8,9 @@ UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads")) with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f: CONFIG: dict = yaml.safe_load(f) -STATUS_BACKEND_PARAMS = CONFIG["status_app"]["backend_params"] \ No newline at end of file +__load_dotenv() + +STATUS_BACKEND_PARAMS = { + **CONFIG["status_app"]["backend_params"], + "url": f"http:{'s' if CONFIG['status_app']['url']['is_https'] else ''}//{os.environ.get('STATUS_BACKEND_BASE_URL', 'localhost')}:{CONFIG['status_app']['url']['port']}" +} diff --git a/docker-compose.yaml b/docker-compose.yaml index b2530d6..cfe0aa9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,36 @@ --- services: - status-backend: - image: harbor.status.im/bi/status-backend:dev - container_name: status-backend - ports: - - 8080:8080 - - 8545:8545 - - 30303:30303 - entrypoint: 'status-backend' - command: '-address 0.0.0.0:8080' - volumes: - - ./data-dir:/data-dir \ No newline at end of file + status-backend: + image: harbor.status.im/bi/status-backend:dev + container_name: status-backend + ports: + - 8080:8080 + - 8545:8545 + - 30303:30303 + entrypoint: 'status-backend' + command: '-address 0.0.0.0:8080' + volumes: + - ./data-dir:/data-dir + networks: + - status-bridge + + status-monitor: + build: + context: . + dockerfile: Dockerfile + container_name: message-processing + depends_on: + - status-backend + env_file: + - .env + environment: + STATUS_BACKEND_HOST: status-backend + volumes: + - ./accounts:/accounts + networks: + - status-bridge + +networks: + status-bridge: + name: status-bridge + driver: bridge diff --git a/upload.py b/upload.py index e0906b3..b54f1d4 100644 --- a/upload.py +++ b/upload.py @@ -3,7 +3,6 @@ import json, datetime, os, time, logging import pandas as pd from pathlib import Path, PosixPath from postgres import Postgres -from dotenv import load_dotenv def get_community_members(file_path: PosixPath) -> pd.DataFrame: """ @@ -118,6 +117,8 @@ def run(username: str, password: str, host: str, database: str, port: str, logge logger.info(f"Created community and member data for {file_path}") for key, value in data.items(): + if not value: + continue data[key] = pd.concat(value, ignore_index=True) file_paths = list(path.rglob("*.json")) @@ -135,9 +136,8 @@ def run(username: str, password: str, host: str, database: str, port: str, logge logger.info(f"Deleted {file_path}") if __name__ == "__main__": - + os.makedirs(constants.UPLOAD_PATH, exist_ok=True) logger = data_utils.get_logger("upload") - load_dotenv() params = { "username": os.getenv("POSTGRES_USERNAME"), "password": os.getenv("POSTGRES_PASSWORD"),