diff --git a/Dockerfile.account_creation b/Dockerfile.account_creation new file mode 100644 index 0000000..cdfc02f --- /dev/null +++ b/Dockerfile.account_creation @@ -0,0 +1,13 @@ +FROM python:3.12-slim +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +# Folder is required only for status-backend and not the code +RUN rm -rf data-dir +# In case if the scripts are ran locally instead of with Docker +RUN rm -rf uploads + +CMD ["python", "create_account.py"] diff --git a/Dockerfile b/Dockerfile.monitoring similarity index 93% rename from Dockerfile rename to Dockerfile.monitoring index 2b1b737..933bbd4 100644 --- a/Dockerfile +++ b/Dockerfile.monitoring @@ -9,5 +9,5 @@ COPY . . RUN rm -rf data-dir # In case if the scripts are ran locally instead of with Docker RUN rm -rf uploads -# Independent scripts + CMD ["sh", "-c", "python upload.py & python download.py"] diff --git a/README.md b/README.md index af33a65..22a49e7 100644 --- a/README.md +++ b/README.md @@ -11,29 +11,60 @@ Monitoring tool for Status App communities - `POSTGRES_HOST` - The Postgres host name that will be remotely connected to. - `POSTGRES_PORT` - The Postgres port that will be remotely connected to. - `STATUS_BACKEND_BASE_URL` (**OPTIONAL**) - The Status Backend URL. If you are running locally you do not need this variable (`localhost` will be automatically set). If you are running it in a Docker container, please set it to `status-backend` (as the `container_name` of the `docker-compose.yaml`). - +- `STATUS_USERNAME` - The Status username that will be used to create an account. This is required if you are running **Dockerfile** for `create_account.py`. +- `STATUS_PASSWORD` - The Status password that will be used to create an account. This is required if you are running **Dockerfile** for `create_account.py`. ## Docker -1. Login to `harbor.status.im`. Your password is your Harbor **CLI secret**. +Login to `harbor.status.im`. Your password is your Harbor **CLI secret**. ```bash docker login harbor.status.im ``` -2. Run `docker-compose.yaml` file +### Account Creation + +To create an account, please make sure you have set the following environment variables: + +- `STATUS_BACKEND_BASE_URL` +- `STATUS_USERNAME` +- `STATUS_PASSWORD` ```bash -docker compose up +docker compose --profile account-creation up +``` + +### Monitoring + +To download and upload messages, please make sure you have set the following environment variables: + +- `POSTGRES_USERNAME` +- `POSTGRES_DATABASE` +- `POSTGRES_HOST` +- `POSTGRES_PORT` +- `STATUS_BACKEND_BASE_URL` + +```bash +docker compose --profile monitoring up +``` + +### Status Backend + +Run Backend independently so you can develop and test locally. + +```bash +docker compose --profile backend up ``` -**Note**: If you want to run Status Backend only, just keep the `status-backend` key in [`services`](./docker-compose.yaml). ## Python -1. Setup environment + +1. Setup environment. [Conda](https://www.anaconda.com/) example: ```bash conda create -n status-monitoring python=3.12 ``` +**Note**: Code has been tested with **Python 3.12**. + 2. Install requirements ```bash @@ -42,10 +73,13 @@ pip install -r requirements.txt **Note**: If you are on Windows, you will have to install `psycopg2` instead of `psycopg2-binary`. -## Files +# Files + +Short explanation of what each runnable file does: - `create_account.py` - create a Status App account for the given `username` and `password`. Example runs: - `python create_account.py -u snt-maxxer -p StatusApp#123` - `python create_account.py --username snt-maxxer --password StatusApp#123` + - `python create_account.py` works if you have added `STATUS_USERNAME` and `STATUS_PASSWORD` in your `.env` file. - `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`. -- `upload.py` - upload data from `download.py` to Postgres +- `upload.py` - upload data from `download.py` to Postgres. diff --git a/config.yaml b/config.yaml index 60bdc51..cca55d3 100644 --- a/config.yaml +++ b/config.yaml @@ -6,7 +6,7 @@ postgres: members: "raw_community_activity" status_app: - bot_name: "snt-maxxer" + bot_name: null # Optional. Can be specified in .env - read README.md channels: - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW url: diff --git a/constants.py b/constants.py index 51e63c0..595e946 100644 --- a/constants.py +++ b/constants.py @@ -10,6 +10,15 @@ with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f: __load_dotenv() +# Make bot_name optional in config.yaml +# You can use .env STATUS_USERNAME as an alternative +__bot_name = CONFIG["status_app"].get("bot_name") +if not __bot_name: + __bot_name = "" + +if len(__bot_name) == 0 and os.environ.get("STATUS_USERNAME"): + CONFIG["status_app"]["bot_name"] = os.environ.get("STATUS_USERNAME") + STATUS_BACKEND_PARAMS = { **CONFIG["status_app"]["backend_params"], "url": f"http:{'s' if CONFIG['status_app']['url']['is_https'] else ''}//{os.environ.get('STATUS_BACKEND_BASE_URL', 'localhost')}:{CONFIG['status_app']['url']['port']}" diff --git a/create_account.py b/create_account.py index a19c468..907f1b7 100644 --- a/create_account.py +++ b/create_account.py @@ -1,24 +1,25 @@ from clients.status_backend import StatusBackend -import os, json, argparse -import constants +import os, json, argparse, logging +import constants, data_utils -def main(username: str, password: str): +def main(username: str, password: str, logger: logging.Logger): os.makedirs(constants.CREDENTIALS_PATH, exist_ok=True) file_path = os.path.join(constants.CREDENTIALS_PATH, f"{username}.json") if os.path.exists(file_path): + logger.info(f"There is a JSON file for {username}. Skipping account creation.") return backend = StatusBackend(**constants.STATUS_BACKEND_PARAMS) info = backend.initialize() - + params = { "display_name": username, "password": password } - + logger.info(f"Creating account for {username}") backend.create_account_and_login(**params) info = backend.wait_for_login() - + logger.info(f"Created account for {username}!") data = { "event": info["event"], "created_at": info["timestamp"], @@ -28,20 +29,27 @@ def main(username: str, password: str): with open(file_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=4, ensure_ascii=False) + logger.info(f"JSON data for {username} saved in {file_path}") backend.logout() + logger.info("Logged out of Status App.") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Login script") parser.add_argument( "-u", "--username", - required=True, help="Status App login username" ) parser.add_argument( "-p", "--password", - required=True, help="Status App local password" ) args = parser.parse_args() - main(args.username, args.password) \ No newline at end of file + + params = { + "username": os.environ.get("STATUS_USERNAME") if not args.username else args.username, + "password": os.environ.get("STATUS_PASSWORD") if not args.password else args.password, + "logger": data_utils.get_logger("create-account") + } + + main(**params) diff --git a/docker-compose.yaml b/docker-compose.yaml index cfe0aa9..1ce9186 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,11 +13,29 @@ services: - ./data-dir:/data-dir networks: - status-bridge - + profiles: + - backend + - monitoring + - account-creation + status-account-creation: + build: + context: . + dockerfile: Dockerfile.account_creation + container_name: account-creation + depends_on: + - status-backend + env_file: + - .env + volumes: + - ./accounts:/app/accounts + networks: + - status-bridge + profiles: + - account-creation status-monitor: build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.monitoring container_name: message-processing depends_on: - status-backend @@ -26,7 +44,9 @@ services: environment: STATUS_BACKEND_HOST: status-backend volumes: - - ./accounts:/accounts + - ./accounts:/app/accounts + profiles: + - monitoring networks: - status-bridge