Initial Dockerfile

- Create `Dockerfile` for `upload.py` and `download.py`. Code and config has been modified to be Docker friendly
- Add `Dockerfile` to `docker-compose.yaml`

**Note**: This version does not work with `create_account.py`

.gitignore

@@ -182,9 +182,9 @@ cython_debug/
 .abstra/
 
 # Visual Studio Code
-#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
 #  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
-#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  and can be added to the global gitignore or merged into this file. However, if you prefer,
 #  you could uncomment the following to ignore the entire vscode folder
 # .vscode/
 
@@ -214,4 +214,5 @@ __marimo__/
 /data-dir
 /uploads
 *.DS_Store
-*.pkl
+*.pkl
+*.dockerignore

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+# Folder is required only for status-backend and not the code
+RUN rm -rf data-dir
+# In case if the scripts are ran locally instead of with Docker
+RUN rm -rf uploads
+# Independent scripts
+CMD ["sh", "-c", "python upload.py & python download.py"]

README.md

@@ -6,13 +6,12 @@ Monitoring tool for Status App communities
 
 ## Environment Variables
 
-```
-POSTGRES_USERNAME
-POSTGRES_PASSWORD
-POSTGRES_DATABASE
-POSTGRES_HOST
-POSTGRES_PORT
-```
+- `POSTGRES_USERNAME` - Postgres username.
+- `POSTGRES_DATABASE` - The database name in the Postgres connection.
+- `POSTGRES_HOST` - The Postgres host name that will be remotely connected to.
+- `POSTGRES_PORT` - The Postgres port that will be remotely connected to.
+- `STATUS_BACKEND_BASE_URL` (**OPTIONAL**) - The Status Backend URL. If you are running locally you do not need this variable (`localhost` will be automatically set). If you are running it in a Docker container, please set it to `status-backend` (as the `container_name` of the `docker-compose.yaml`).
+
 
 ## Docker
 
@@ -27,6 +26,7 @@ docker login harbor.status.im
 ```bash
 docker compose up
 ```
+**Note**: If you want to run Status Backend only, just keep the `status-backend` key in [`services`](./docker-compose.yaml).
 
 ## Python
 1. Setup environment
@@ -39,6 +39,7 @@ conda create -n status-monitoring python=3.12
 ```bash
 pip install -r requirements.txt
 ```
+
 **Note**: If you are on Windows, you will have to install `psycopg2` instead of `psycopg2-binary`.
 
 ## Files
@@ -47,4 +48,4 @@ pip install -r requirements.txt
   - `python create_account.py -u snt-maxxer -p StatusApp#123`
   - `python create_account.py --username snt-maxxer --password StatusApp#123`
 - `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`.
-- `upload.py` - upload data from `download.py` to Postgres
+- `upload.py` - upload data from `download.py` to Postgres

config.yaml

@@ -1,19 +1,20 @@
 postgres:
-  schema: "status_app_monitoring"
-  tables:
-    messages: "raw_messages"
-    community: "raw_community_info"
-    members: "raw_community_activity"
+    schema: "status_app_monitoring"
+    tables:
+        messages: "raw_messages"
+        community: "raw_community_info"
+        members: "raw_community_activity"
 
 status_app:
-  bot_name: "snt-maxxer"
-  channels:
-    - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW
-
-  backend_params:
-    url: "http://localhost:8080"
-    logLevel: "INFO"
-    data_dir: "./data-dir"
+    bot_name: "snt-maxxer"
+    channels:
+        - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW
+    url:
+        port: 8080
+        is_https: false
+    backend_params:
+        logLevel: "INFO"
+        data_dir: "./data-dir"
 
 sleep:
     # Values must be in MINUTES

constants.py

@@ -1,5 +1,6 @@
 import os
 import yaml
+from dotenv import load_dotenv as __load_dotenv
 
 CREDENTIALS_PATH = os.path.join(os.path.dirname(__file__), "accounts")
 UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads"))
@@ -7,4 +8,9 @@ UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads"))
 with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f:
     CONFIG: dict = yaml.safe_load(f)
 
-STATUS_BACKEND_PARAMS = CONFIG["status_app"]["backend_params"]
+__load_dotenv()
+
+STATUS_BACKEND_PARAMS = {
+    **CONFIG["status_app"]["backend_params"],
+    "url": f"http:{'s' if CONFIG['status_app']['url']['is_https']  else ''}//{os.environ.get('STATUS_BACKEND_BASE_URL', 'localhost')}:{CONFIG['status_app']['url']['port']}"
+}

docker-compose.yaml

@@ -1,13 +1,36 @@
 ---
 services:
-  status-backend:
-    image: harbor.status.im/bi/status-backend:dev
-    container_name: status-backend
-    ports:
-      - 8080:8080
-      - 8545:8545
-      - 30303:30303
-    entrypoint: 'status-backend'
-    command: '-address 0.0.0.0:8080'
-    volumes:
-      - ./data-dir:/data-dir
+    status-backend:
+        image: harbor.status.im/bi/status-backend:dev
+        container_name: status-backend
+        ports:
+            - 8080:8080
+            - 8545:8545
+            - 30303:30303
+        entrypoint: 'status-backend'
+        command: '-address 0.0.0.0:8080'
+        volumes:
+            - ./data-dir:/data-dir
+        networks:
+            - status-bridge
+
+    status-monitor:
+        build:
+            context: .
+            dockerfile: Dockerfile
+        container_name: message-processing
+        depends_on:
+            - status-backend
+        env_file:
+            - .env
+        environment:
+            STATUS_BACKEND_HOST: status-backend
+        volumes:
+            - ./accounts:/accounts
+        networks:
+            - status-bridge
+
+networks:
+    status-bridge:
+        name: status-bridge
+        driver: bridge

upload.py

@@ -3,7 +3,6 @@ import json, datetime, os, time, logging
 import pandas as pd
 from pathlib import Path, PosixPath
 from postgres import Postgres
-from dotenv import load_dotenv
 
 def get_community_members(file_path: PosixPath) -> pd.DataFrame:
     """
@@ -118,6 +117,8 @@ def run(username: str, password: str, host: str, database: str, port: str, logge
         logger.info(f"Created community and member data for {file_path}")
 
     for key, value in data.items():
+        if not value:
+            continue
         data[key] = pd.concat(value, ignore_index=True)
 
     file_paths = list(path.rglob("*.json"))
@@ -135,9 +136,8 @@ def run(username: str, password: str, host: str, database: str, port: str, logge
         logger.info(f"Deleted {file_path}")
 
 if __name__ == "__main__":
-
+    os.makedirs(constants.UPLOAD_PATH, exist_ok=True)
     logger = data_utils.get_logger("upload")
-    load_dotenv()
     params = {
         "username": os.getenv("POSTGRES_USERNAME"),
         "password": os.getenv("POSTGRES_PASSWORD"),