download: Batch download

- Create community info snapshots (`.pkl` file)
- Create messages file that will be uploaded to Postgres (`.json` file)
- Create configurable file to keep track of changes on GitHub

.gitignore

@@ -211,4 +211,5 @@ __marimo__/
 .vscode/
 *.json
 *.ipynb
-/data-dir
+/data-dir
+/uploads

README.md

@@ -32,7 +32,7 @@ pip install -r requirements.txt
 
 ## Files
 
-- `create_account.py` - create a Status App account for the given `username` and `password`. Example run:
+- `create_account.py` - create a Status App account for the given `username` and `password`. Example runs:
 
 ```bash
 python create_account.py -u snt-maxxer -p StatusApp#123
@@ -40,4 +40,6 @@ python create_account.py -u snt-maxxer -p StatusApp#123
 
 ```bash
 python create_account.py --username snt-maxxer --password StatusApp#123
-```
+```
+
+- `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`.

config.yaml

@@ -0,0 +1,15 @@
+postgres:
+  schema: "status_app_monitoring"
+  tables:
+    messages: "raw_messages"
+    community: "community_info"
+
+status_app:
+  bot_name: "snt-maxxer"
+  channels:
+    - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW
+  
+  backend_params:
+    url: "http://localhost:8080"
+    logLevel: "INFO"
+    data_dir: "./data-dir"

constants.py

@@ -1,9 +1,10 @@
 import os
+import yaml
 
 CREDENTIALS_PATH = os.path.join(os.path.dirname(__file__), "accounts")
+UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads"))
 
-STATUS_BACKEND_PARAMS = {
-    "url": "http://localhost:8080",
-    "logLevel": "INFO",
-    "data_dir": "./data-dir" # <- Used in the container
-}
+with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f:
+    CONFIG: dict = yaml.safe_load(f)
+
+STATUS_BACKEND_PARAMS = CONFIG["status_app"]["backend_params"]

data_utils.py

@@ -2,7 +2,7 @@
 Function that simplify the Status Backend data extraction process
 """
 from clients.status_backend import StatusBackend
-import os, json, datetime
+import os, json, datetime, json
 import constants
 
 def login(backend: StatusBackend, username: str, is_chat: bool = True) -> dict:
@@ -64,8 +64,10 @@ def get_community_info(backend: StatusBackend, url: str) -> dict:
         raise Exception(f"No community info found for {url}")
 
     to_datetime = lambda key: datetime.datetime.fromtimestamp(community_info[key]) if key in community_info else None
+    extract_timestamp = datetime.datetime.now()
     data = {
         "community_id": community_info["id"],
+        "community_name": community_info["name"],
         "url": url,
         "verified": community_info["verified"],
         "description": community_info["description"],
@@ -76,11 +78,22 @@ def get_community_info(backend: StatusBackend, url: str) -> dict:
         "joined_timestamp": to_datetime("joinedAt"),
         "requested_timestamp": to_datetime("requestedToJoinAt"),
         "encrypted": community_info["encrypted"],
-        "members": len(community_info["members"].keys()),
+        "members": {
+            "total": len(community_info["members"].keys()),
+            "info": [
+                {
+                    "member_id": member_id, 
+                    "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None,
+                    "extract_timestamp": extract_timestamp
+                } 
+                for member_id, info in community_info["members"].items()
+            ]
+        },
         "channels": [
             {
                 "community_id": community_info["id"],
-                "channel_id": chat_info["id"], 
+                "channel_id": chat_info["id"],
+                "chat_id": community_info["id"] + chat_info["id"],
                 "category_id": chat_info["categoryID"] if len(chat_info["categoryID"]) > 0 else None,
                 "channel_name": chat_info["name"], 
                 "description": chat_info["description"],
@@ -116,7 +129,7 @@ def get_contacts(backend: StatusBackend) -> list[dict]:
 
 
 
-def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination: int = 100, batch_size: int = 10):
+def save_messages(backend: StatusBackend, chat_id: str, folder: str, community_info: dict, pagination: int = 100, batch_size: int = 10):
     """
     Get all of the mesages from a chat.
     NOTE: You have to be logged in to Status app already!
@@ -127,17 +140,36 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination:
         - `pagination` - how many results to get per `.chat_messages` call
         - `batch_size` - the number of messages that will be turned into a batch
     """
+    def save_batch(messages: list[str], folder: str):
+        timestamp = datetime.datetime.now().timestamp()
+
+        file_path = os.path.join(folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".json")
+        data = {
+            "metadata": {
+                "file_path": file_path,
+                "created_at": timestamp,
+                "total_messages": len(messages),
+                "earliest_msg_timestamp": min(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"],
+                "latest_msg_timestamp": max(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"],
+            },
+            "messages": messages,
+        }
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=4, ensure_ascii=False)
+
+    os.makedirs(folder, exist_ok=True)
     messages = []
     cursor = None
     mappings = {
         "id": "message_id",
-        "chatId": "chat_id",
+        "chatId": "channel_id",
         "timestamp": "sent_timestamp",
         "compressedKey": "from_key",
         "emojiHash": "from_emojis",
         "parsedText": "parsed_text",
         "text": "markdown_text",
         "editedAt": "edited_timestamp",
+        "links": "links"
     }
     finished = False
     while not finished:
@@ -152,10 +184,13 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination:
 
         messages += [
             {
-                target_key: msg[msg_key]
-                for msg_key, target_key in mappings.items()
-                if msg_key in msg
-            } 
+                "community_id": community_info["community_id"],
+                "channel_id": community_info["channel_id"],
+                "chat_id": community_info["chat_id"],
+                "channel_category_id": community_info["category_id"],
+                **{target_key: msg[msg_key] for msg_key, target_key in mappings.items() if msg_key in msg},
+                "extracted_at": datetime.datetime.now().timestamp()
+            }
             for msg in chat["messages"]
         ]
         if len(chat["cursor"]) > 0:
@@ -163,5 +198,10 @@ def get_messages(backend: StatusBackend, chat_id: str, folder: str, pagination:
 
         finished = len(chat["cursor"]) == 0
         
+        if len(messages) >= batch_size:
+            save_batch(messages, folder)
+            messages = []
         
-    return messages
+    
+    if messages:
+        save_batch(messages, folder)

download.py

@@ -0,0 +1,25 @@
+from clients.status_backend import StatusBackend
+import constants, data_utils
+import os, datetime, pickle
+
+if __name__ == "__main__":
+
+    backend = StatusBackend(**constants.STATUS_BACKEND_PARAMS)
+    info = data_utils.login(backend, constants.CONFIG["status_app"]["bot_name"])
+    
+    message_folder = os.path.join(constants.UPLOAD_PATH, "messages")
+    community_folder = os.path.join(constants.UPLOAD_PATH, "channel_info")
+
+    for channel_url in constants.CONFIG["status_app"]["channels"]:
+        community = data_utils.get_community_info(backend, channel_url)
+        current_community_folder = os.path.join(community_folder, community["community_id"])
+        os.makedirs(current_community_folder, exist_ok=True)
+        file_path = os.path.join(current_community_folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".pkl")
+        
+        with open(file_path, "wb") as f:
+            pickle.dump(community, f)
+
+        for channel in community["channels"]:
+            data_utils.save_messages(backend, channel["chat_id"], message_folder, channel)
+
+    backend.logout()

requirements.txt

@@ -12,4 +12,6 @@ web3~=7.9.0
 matplotlib>=3.5.0
 eth-typing~=5.2.1
 faker~=37.6.0
-qrcode
+qrcode
+pandas
+pyyaml