From ccfed7fc7c82ade5cc801f3c860f5845ecf7a6b5 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Fri, 27 Feb 2026 20:49:32 +0200 Subject: [PATCH] download: Batch download - Create community info snapshots (`.pkl` file) - Create messages file that will be uploaded to Postgres (`.json` file) - Create configurable file to keep track of changes on GitHub --- .gitignore | 3 ++- README.md | 6 +++-- config.yaml | 15 ++++++++++++ constants.py | 11 +++++---- data_utils.py | 60 ++++++++++++++++++++++++++++++++++++++++-------- download.py | 25 ++++++++++++++++++++ requirements.txt | 4 +++- 7 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 config.yaml create mode 100644 download.py diff --git a/.gitignore b/.gitignore index f22987e..9ab1965 100644 --- a/.gitignore +++ b/.gitignore @@ -211,4 +211,5 @@ __marimo__/ .vscode/ *.json *.ipynb -/data-dir \ No newline at end of file +/data-dir +/uploads diff --git a/README.md b/README.md index 13af1ce..3835421 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ pip install -r requirements.txt ## Files -- `create_account.py` - create a Status App account for the given `username` and `password`. Example run: +- `create_account.py` - create a Status App account for the given `username` and `password`. Example runs: ```bash python create_account.py -u snt-maxxer -p StatusApp#123 @@ -40,4 +40,6 @@ python create_account.py -u snt-maxxer -p StatusApp#123 ```bash python create_account.py --username snt-maxxer --password StatusApp#123 -``` \ No newline at end of file +``` + +- `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`. \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..18afa35 --- /dev/null +++ b/config.yaml @@ -0,0 +1,15 @@ +postgres: + schema: "status_app_monitoring" + tables: + messages: "raw_messages" + community: "community_info" + +status_app: + bot_name: "snt-maxxer" + channels: + - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW + + backend_params: + url: "http://localhost:8080" + logLevel: "INFO" + data_dir: "./data-dir" \ No newline at end of file diff --git a/constants.py b/constants.py index 5084f53..656cc5b 100644 --- a/constants.py +++ b/constants.py @@ -1,9 +1,10 @@ import os +import yaml CREDENTIALS_PATH = os.path.join(os.path.dirname(__file__), "accounts") +UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads")) -STATUS_BACKEND_PARAMS = { - "url": "http://localhost:8080", - "logLevel": "INFO", - "data_dir": "./data-dir" # <- Used in the container -} +with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f: + CONFIG: dict = yaml.safe_load(f) + +STATUS_BACKEND_PARAMS = CONFIG["status_app"]["backend_params"] \ No newline at end of file diff --git a/data_utils.py b/data_utils.py index dde5d80..aed5456 100644 --- a/data_utils.py +++ b/data_utils.py @@ -2,7 +2,7 @@ Function that simplify the Status Backend data extraction process """ from clients.status_backend import StatusBackend -import os, json, datetime +import os, json, datetime, json import constants def login(backend: StatusBackend, username: str, is_chat: bool = True) -> dict: @@ -64,8 +64,10 @@ def get_community_info(backend: StatusBackend, url: str) -> dict: raise Exception(f"No community info found for {url}") to_datetime = lambda key: datetime.datetime.fromtimestamp(community_info[key]) if key in community_info else None + extract_timestamp = datetime.datetime.now() data = { "community_id": community_info["id"], + "community_name": community_info["name"], "url": url, "verified": community_info["verified"], "description": community_info["description"], @@ -76,11 +78,22 @@ def get_community_info(backend: StatusBackend, url: str) -> dict: "joined_timestamp": to_datetime("joinedAt"), "requested_timestamp": to_datetime("requestedToJoinAt"), "encrypted": community_info["encrypted"], - "members": len(community_info["members"].keys()), + "members": { + "total": len(community_info["members"].keys()), + "info": [ + { + "member_id": member_id, + "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None, + "extract_timestamp": extract_timestamp + } + for member_id, info in community_info["members"].items() + ] + }, "channels": [ { "community_id": community_info["id"], - "channel_id": chat_info["id"], + "channel_id": chat_info["id"], + "chat_id": community_info["id"] + chat_info["id"], "category_id": chat_info["categoryID"] if len(chat_info["categoryID"]) > 0 else None, "channel_name": chat_info["name"], "description": chat_info["description"], @@ -116,7 +129,7 @@ def get_contacts(backend: StatusBackend) -> list[dict]: -def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination: int = 100, batch_size: int = 10): +def save_messages(backend: StatusBackend, chat_id: str, folder: str, community_info: dict, pagination: int = 100, batch_size: int = 10): """ Get all of the mesages from a chat. NOTE: You have to be logged in to Status app already! @@ -127,17 +140,36 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination: - `pagination` - how many results to get per `.chat_messages` call - `batch_size` - the number of messages that will be turned into a batch """ + def save_batch(messages: list[str], folder: str): + timestamp = datetime.datetime.now().timestamp() + + file_path = os.path.join(folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".json") + data = { + "metadata": { + "file_path": file_path, + "created_at": timestamp, + "total_messages": len(messages), + "earliest_msg_timestamp": min(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"], + "latest_msg_timestamp": max(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"], + }, + "messages": messages, + } + with open(file_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=4, ensure_ascii=False) + + os.makedirs(folder, exist_ok=True) messages = [] cursor = None mappings = { "id": "message_id", - "chatId": "chat_id", + "chatId": "channel_id", "timestamp": "sent_timestamp", "compressedKey": "from_key", "emojiHash": "from_emojis", "parsedText": "parsed_text", "text": "markdown_text", "editedAt": "edited_timestamp", + "links": "links" } finished = False while not finished: @@ -152,10 +184,13 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination: messages += [ { - target_key: msg[msg_key] - for msg_key, target_key in mappings.items() - if msg_key in msg - } + "community_id": community_info["community_id"], + "channel_id": community_info["channel_id"], + "chat_id": community_info["chat_id"], + "channel_category_id": community_info["category_id"], + **{target_key: msg[msg_key] for msg_key, target_key in mappings.items() if msg_key in msg}, + "extracted_at": datetime.datetime.now().timestamp() + } for msg in chat["messages"] ] if len(chat["cursor"]) > 0: @@ -163,5 +198,10 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination: finished = len(chat["cursor"]) == 0 + if len(messages) >= batch_size: + save_batch(messages, folder) + messages = [] - return messages \ No newline at end of file + + if messages: + save_batch(messages, folder) \ No newline at end of file diff --git a/download.py b/download.py new file mode 100644 index 0000000..38ee64a --- /dev/null +++ b/download.py @@ -0,0 +1,25 @@ +from clients.status_backend import StatusBackend +import constants, data_utils +import os, datetime, pickle + +if __name__ == "__main__": + + backend = StatusBackend(**constants.STATUS_BACKEND_PARAMS) + info = data_utils.login(backend, constants.CONFIG["status_app"]["bot_name"]) + + message_folder = os.path.join(constants.UPLOAD_PATH, "messages") + community_folder = os.path.join(constants.UPLOAD_PATH, "channel_info") + + for channel_url in constants.CONFIG["status_app"]["channels"]: + community = data_utils.get_community_info(backend, channel_url) + current_community_folder = os.path.join(community_folder, community["community_id"]) + os.makedirs(current_community_folder, exist_ok=True) + file_path = os.path.join(current_community_folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".pkl") + + with open(file_path, "wb") as f: + pickle.dump(community, f) + + for channel in community["channels"]: + data_utils.save_messages(backend, channel["chat_id"], message_folder, channel) + + backend.logout() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a0008fe..55e7618 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,6 @@ web3~=7.9.0 matplotlib>=3.5.0 eth-typing~=5.2.1 faker~=37.6.0 -qrcode \ No newline at end of file +qrcode +pandas +pyyaml \ No newline at end of file