From bcb34da804f564a6d92700ed53e0563a7a306686 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Thu, 19 Mar 2026 22:47:35 +0200 Subject: [PATCH 1/6] bot: Status App Monitoring Related to: - https://github.com/status-im/status-bot/issues/14 - https://github.com/status-im/status-bot/issues/4 - https://github.com/status-im/data-docs/issues/153 Notes: - SImplified `Account` class is used instead of initial `StatusBackend` - Add `Account` requirements.txt - Fixed backup load bug. Initially wrong event was monitored and it would wait forever. - Latest Status Backend Dockerfile is pulled from GitHub instead of Harbor - Postgres schema is created if it doesn't exist --- Dockerfile | 11 +- README.md | 43 +- bot/__init__.py | 1 + bot/account.py | 4 +- bot/requirements.txt | 2 + clients/anvil.py | 76 -- clients/api.py | 70 -- clients/connector.py | 118 --- clients/contract_deployers/communities.py | 15 - clients/contract_deployers/multicall3.py | 18 - clients/contract_deployers/snt.py | 33 - clients/expvar.py | 138 ---- clients/foundry.py | 190 ----- clients/gorush_stub.py | 160 ---- clients/metrics.py | 570 ------------- clients/push_notification_server.py | 17 - clients/rpc.py | 42 - clients/services/accounts.py | 219 ----- clients/services/appgeneral.py | 17 - clients/services/connector.py | 24 - clients/services/eth.py | 12 - clients/services/multiaccounts.py | 17 - clients/services/newsfeed.py | 34 - clients/services/service.py | 12 - clients/services/settings.py | 16 - clients/services/sharedurls.py | 47 -- clients/services/wakuext.py | 777 ------------------ clients/services/wallet.py | 94 --- clients/signals.py | 226 ----- clients/status_backend.py | 524 ------------ clients/statusgo_container.py | 479 ----------- config.yaml | 19 +- constants.py | 25 - create_account.py | 55 -- data_utils.py | 270 ------ docker-compose.yaml | 6 +- download.py | 54 -- entrypoint.sh | 8 - monitor.py | 189 +++++ postgres.py | 48 +- requirements.txt | 17 +- resources/__init__.py | 0 resources/constants.py | 142 ---- resources/enums.py | 56 -- resources/test_data/__init__.py | 1 - resources/test_data/mnemonic_12.py | 58 -- resources/test_data/mnemonic_15.py | 39 - resources/test_data/mnemonic_24.py | 39 - resources/test_data/profile_showcase_utils.py | 93 --- resources/utils.py | 20 - upload.py | 154 ---- utils/__init__.py | 0 utils/config.py | 37 - utils/fake.py | 54 -- utils/image_utils.py | 6 - utils/keys.py | 18 - utils/retry_utils.py | 15 - utils/wallet_utils.py | 152 ---- 58 files changed, 267 insertions(+), 5314 deletions(-) create mode 100644 bot/requirements.txt delete mode 100644 clients/anvil.py delete mode 100644 clients/api.py delete mode 100644 clients/connector.py delete mode 100644 clients/contract_deployers/communities.py delete mode 100644 clients/contract_deployers/multicall3.py delete mode 100644 clients/contract_deployers/snt.py delete mode 100644 clients/expvar.py delete mode 100644 clients/foundry.py delete mode 100644 clients/gorush_stub.py delete mode 100644 clients/metrics.py delete mode 100644 clients/push_notification_server.py delete mode 100644 clients/rpc.py delete mode 100644 clients/services/accounts.py delete mode 100644 clients/services/appgeneral.py delete mode 100644 clients/services/connector.py delete mode 100644 clients/services/eth.py delete mode 100644 clients/services/multiaccounts.py delete mode 100644 clients/services/newsfeed.py delete mode 100644 clients/services/service.py delete mode 100644 clients/services/settings.py delete mode 100644 clients/services/sharedurls.py delete mode 100644 clients/services/wakuext.py delete mode 100644 clients/services/wallet.py delete mode 100644 clients/signals.py delete mode 100644 clients/status_backend.py delete mode 100644 clients/statusgo_container.py delete mode 100644 constants.py delete mode 100644 create_account.py delete mode 100644 data_utils.py delete mode 100644 download.py delete mode 100644 entrypoint.sh create mode 100644 monitor.py delete mode 100644 resources/__init__.py delete mode 100644 resources/constants.py delete mode 100644 resources/enums.py delete mode 100644 resources/test_data/__init__.py delete mode 100644 resources/test_data/mnemonic_12.py delete mode 100644 resources/test_data/mnemonic_15.py delete mode 100644 resources/test_data/mnemonic_24.py delete mode 100644 resources/test_data/profile_showcase_utils.py delete mode 100644 resources/utils.py delete mode 100644 upload.py delete mode 100644 utils/__init__.py delete mode 100644 utils/config.py delete mode 100644 utils/fake.py delete mode 100644 utils/image_utils.py delete mode 100644 utils/keys.py delete mode 100644 utils/retry_utils.py delete mode 100644 utils/wallet_utils.py diff --git a/Dockerfile b/Dockerfile index c0133bc..73f6782 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,12 @@ FROM python:3.12-slim WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - COPY . . -# Folder is required only for status-backend and not the code -RUN rm -rf data-dir # In case if the scripts are ran locally instead of with Docker RUN rm -rf uploads -RUN chmod +x entrypoint.sh +RUN pip install --no-cache-dir -r requirements.txt \ + && if [ -f bot/requirements.txt ]; then pip install --no-cache-dir -r bot/requirements.txt; fi -ENTRYPOINT ["bash", "/app/entrypoint.sh"] +ENTRYPOINT ["python", "monitor.py"] CMD [] diff --git a/README.md b/README.md index 0015e19..d69fc98 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # [Status App Monitoring](https://status.app/) -Monitoring tool for Status App communities +Monitoring tool for Status App communities. # Setup @@ -11,9 +11,9 @@ Monitoring tool for Status App communities - `POSTGRES_DATABASE` - The database name in the Postgres connection. - `POSTGRES_HOST` - The Postgres host name that will be remotely connected to. - `POSTGRES_PORT` - The Postgres port that will be remotely connected to. -- `STATUS_BACKEND_BASE_URL` (**OPTIONAL**) - The Status Backend URL. If you are running locally you do not need this variable (`localhost` will be automatically set). If you are running it in a Docker container, please set it to `status-backend` (as the `container_name` of the `docker-compose.yaml`). -- `STATUS_USERNAME` - The Status username that will be used to create an account. This is required if you are running **Dockerfile** for `create_account.py`. -- `STATUS_PASSWORD` - The Status password that will be used to create an account. This is required if you are running **Dockerfile** for `create_account.py`. +- `STATUS_DISPLAY_NAME` - The Status display name that will be used to create an account. +- `STATUS_PASSWORD` - The Status password that will be used to create an account. +- `STATUS_MNEMONIC` - The mnemonic used to recover the account. If passed a `.bkp` file will be loaded as well. Use this when you want to login to a bot account via Status App, join a community / leave community and export the `.bkp` file. ## Docker deployement @@ -22,10 +22,10 @@ You can use the `docker-compose.yaml` to run the project. Example of `.env` file to use ``` # Status Backend connection -STATUS_BACKEND_BASE_URL=status-backend -STATUS_USERNAME=bot-status -STATUS_PASSWORD=ChangeThisPassword -INIT_ACCOUNT=true +STATUS_DISPLAY_NAME = "bot-status" +STATUS_PASSWORD = "ChangeThisPassword" +STATUS_MNEMONIC= "test test test test test test test test test test test test" + # Database config POSTGRES_HOST=database POSTGRES_PORT=5432 @@ -34,16 +34,6 @@ POSTGRES_USERNAME=status POSTGRES_PASSWORD=ChangeThisOneAlso ``` -` INIT_ACCOUNT=true` will create a new account, if set to false, it will the credentials in the `accounts/ ` directory. - -### Status Backend - -Run Backend independently so you can develop and test locally. - -```bash -docker compose run status-backend up -d -``` - ## Python 1. Setup environment. [Conda](https://www.anaconda.com/) example: @@ -61,13 +51,14 @@ pip install -r requirements.txt **Note**: If you are on Windows, you will have to install `psycopg2` instead of `psycopg2-binary`. +# Backups + +If you have already created a Status account and want to use it with it's current data, please make sure you export the `.bkp` file and put it in folder **backups** and have the following `.env` variables: + +- `STATUS_DISPLAY_NAME` +- `STATUS_PASSWORD` +- `STATUS_MNEMONIC` + # Files -Short explanation of what each runnable file does: - -- `create_account.py` - create a Status App account for the given `username` and `password`. Example runs: - - `python create_account.py -u snt-maxxer -p StatusApp#123` - - `python create_account.py --username snt-maxxer --password StatusApp#123` - - `python create_account.py` works if you have added `STATUS_USERNAME` and `STATUS_PASSWORD` in your `.env` file. -- `download.py` - download all messages and overall community info from the specified Status App channels in `config.yaml`. -- `upload.py` - upload data from `download.py` to Postgres. +- `monitor.py` - Status community message monitoring. It will download and upload messages in parallel. diff --git a/bot/__init__.py b/bot/__init__.py index 301bf3a..d607e0e 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -1 +1,2 @@ from .account import Account +from .logger import Logger diff --git a/bot/account.py b/bot/account.py index 16bc8af..daf9a80 100644 --- a/bot/account.py +++ b/bot/account.py @@ -310,7 +310,7 @@ class Account: return self.__signal @property - def communities(self) -> dict[str, dict]: + def communities(self) -> list[dict]: """ Get the communities that the bot is in. NOTE: We do not use internal state so we can get dynamic values such as: @@ -597,7 +597,7 @@ class Account: response = requests.post(self.urls["http"]["load_backup"], json=params) error: str = response.json().get("error", "") if len(error) == 0: - self.__signal.get("history.request.completed") + self.__signal.get("messages.new") self.logger.info(f"Successfully loaded file!") break diff --git a/bot/requirements.txt b/bot/requirements.txt new file mode 100644 index 0000000..d1b7668 --- /dev/null +++ b/bot/requirements.txt @@ -0,0 +1,2 @@ +requests +websocket-client diff --git a/clients/anvil.py b/clients/anvil.py deleted file mode 100644 index 202ee81..0000000 --- a/clients/anvil.py +++ /dev/null @@ -1,76 +0,0 @@ -import logging -import time -import docker - -from utils.config import Config -from tenacity import retry, wait_fixed, stop_after_attempt -from web3 import Web3 -from web3.types import ( - TxData, - TxReceipt, - RPCEndpoint, -) -from eth_typing import ( - HexStr, -) -from typing import ( - Union, -) -from hexbytes import HexBytes - - -class Anvil(Web3): - - def __init__(self): - self.docker_client = docker.from_env() - self.docker_project_name = Config.docker_project_name - self.network_name = f"{self.docker_project_name}_default" - - container_name_prefix = f"{self.docker_project_name}-anvil" - self.container_name = self.find_container_name(self.network_name, container_name_prefix) - - if not self.container_name: - raise Exception("Anvil container not found") - self.container = self.docker_client.containers.get(self.container_name) - network_info = self.container.attrs["NetworkSettings"]["Ports"].get("8545/tcp", []) - if not network_info: - raise Exception("Anvil exposed port not found") - self.ip = network_info[0]["HostIp"] - self.port = network_info[0]["HostPort"] - self.anvil_url = f"http://{self.ip}:{self.port}" - logging.info(f"Anvil URL: {self.anvil_url}") - Web3.__init__(self, Web3.HTTPProvider(self.anvil_url)) - self.wait_for_healthy() - - @retry(stop=stop_after_attempt(10), wait=wait_fixed(0.1), reraise=True) - def find_container_name(self, network_name, searched_container): - network = self.docker_client.networks.get(network_name) - - for container in network.containers: - container_name = container.name - if container_name is not None and searched_container in container_name: - return container_name - - return None - - def wait_for_healthy(self, timeout=10): - start_time = time.time() - while time.time() - start_time <= timeout: - if self.is_connected(show_traceback=True): - logging.info(f"Anvil is healthy after {time.time() - start_time} seconds") - return - else: - time.sleep(0.1) - raise TimeoutError(f"Anvil was not healthy after {timeout} seconds") - - def get_transaction(self, tx_hash: str) -> TxData: - return self.eth.get_transaction(HexStr(tx_hash)) - - def transaction_receipt(self, tx_hash: str) -> TxReceipt: - return self.eth.get_transaction_receipt(HexStr(tx_hash)) - - def send_raw_transaction(self, transaction: Union[HexStr, bytes]) -> HexBytes: - return self.eth.send_raw_transaction(transaction) - - def set_balance(self, address: str, raw_amount: int): - return self.provider.make_request(RPCEndpoint("anvil_setBalance"), [address, hex(raw_amount)]) diff --git a/clients/api.py b/clients/api.py deleted file mode 100644 index 8304585..0000000 --- a/clients/api.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import logging -from json import JSONDecodeError - -import requests - - -class ApiError(Exception): - def __init__(self, message, *, method=None, payload=None): - super().__init__(message) - self.method = method - self.payload = payload - - -class ApiHTTPError(ApiError): - def __init__(self, message, *, method=None, status=None, payload=None): - super().__init__(message, method=method, payload=payload) - self.status = status - - -class ApiDecodeError(ApiError): - pass - - -class ApiResponseError(ApiError): - pass - - -class ApiClient: - def __init__(self, api_url, client=requests.Session()): - self.client = client - self.api_url = api_url - - def method_url(self, method): - return f"{self.api_url}/{method}" - - def api_request(self, method, data, url=None, quiet=False, **kwargs): - url = url if url else self.api_url - url = f"{url}/{method}" if method else url - if not quiet: - logging.debug(f"Sending POST request to url {url} with data: {json.dumps(data, sort_keys=True)}") - response = self.client.post(url, json=data, **kwargs) - - if response.status_code != 200: - raise ApiHTTPError( - f"HTTP {response.status_code}", - method=method, - status=response.status_code, - payload=getattr(response, "text", None), - ) - - if not response.content: - raise ApiHTTPError("Empty response body", method=method, status=response.status_code) - - if not quiet: - logging.debug(f"Got response: {response.content}") - return response - - def api_request_json(self, method, data, **kwargs): - response = self.api_request(method, data, **kwargs) - try: - json_response = response.json() - except JSONDecodeError: - raise ApiDecodeError("Invalid JSON in response", method=method, payload=response.content) - - err = json_response.get("error") - if err: - raise ApiResponseError(str(err), method=method, payload=json_response) - - return json_response diff --git a/clients/connector.py b/clients/connector.py deleted file mode 100644 index cddbb14..0000000 --- a/clients/connector.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import logging -import requests -from typing import Any, Dict -import websocket -from websocket import WebSocket -from websocket import create_connection - - -class ConnectorApiError(Exception): - def __init__(self, message, code): - super().__init__(message) - self.code = code - - -class ConnectorClient: - def __init__(self, url: str): - self.url = url - self.ws_conn: WebSocket | None = None - self._request_id = 0 - self.wrapped_request_id = 0 - self.name = "" - - @property - def request_id(self) -> int: - self._request_id += 1 - return self._request_id - - def connect(self): - http_url = self.url.replace("ws", "http") - logging.debug(f"ConnectorClient: sending initial HEAD request to {http_url}") - response = requests.head(http_url, timeout=5) - assert response.status_code == 404 - - logging.debug(f"ConnectorClient: connecting to {self.url}") - origin = "https://www.example.com" # At the moment all origins are allowed - self.ws_conn = create_connection(self.url, origin=origin) - assert self.ws_conn is not None - assert self.ws_conn.sock is not None - - # Use a random name for dApp name - port = self.ws_conn.sock.getsockname()[1] - self.name = f"status-go-functional-tests-{port}" - - def disconnect(self): - if self.ws_conn is not None: - self.ws_conn.close() - - def eth_chain_id(self): - self._send("eth_chainId") - - def eth_accounts(self): - self._send("eth_accounts") - - def eth_request_accounts(self): - self._send("eth_requestAccounts") - - def eth_block_number(self): - self._send("eth_blockNumber") - - def eth_get_balance(self, address: str): - self._send("eth_getBalance", [address, "latest"]) - - def eth_get_transaction_count(self, address: str): - self._send("eth_getTransactionCount", [address, "latest"]) - - def eth_call(self, call_object: Dict[str, Any]): - self._send("eth_call", [call_object, "latest"]) - - def eth_estimate_gas(self, tx_object: Dict[str, Any]): - self._send("eth_estimateGas", [tx_object]) - - def eth_get_transaction_receipt(self, tx_hash: str): - self._send("eth_getTransactionReceipt", [tx_hash]) - - def eth_send_transaction(self, tx_object: Dict[str, Any]): - self._send("eth_sendTransaction", [tx_object]) - - def wallet_switch_ethereum_chain(self, chain_id: int): - self._send("wallet_switchEthereumChain", [{"chainId": hex(chain_id)}]) - - def wallet_revoke_permissions(self): - self._send("wallet_revokePermissions") - - def _send(self, method, params=None): - assert self.ws_conn is not None - - request_id = self.request_id - request = { - "jsonrpc": "2.0", - "id": request_id, - "name": self.name, - "url": "http://localhost/", - "method": method, - "clientId": "tests-functional", - } - if params is not None: - request["params"] = params - - wrapped_request = { - "jsonrpc": "2.0", - "id": request_id, - "method": "connector_callRPC", - "params": [json.dumps(request)], - } - - logging.debug(f"Sending Connector request with data: {json.dumps(wrapped_request, sort_keys=True)}") - self.ws_conn.send(json.dumps(wrapped_request), websocket.ABNF.OPCODE_TEXT) - - def receive(self): - assert self.ws_conn is not None - response = self.ws_conn.recv() - logging.debug(f"Got Connector response: {json.dumps(response, sort_keys=True)}") - response = json.loads(response) - error = response.get("error") - if error is not None: - raise ConnectorApiError(error["message"], error["code"]) - return response diff --git a/clients/contract_deployers/communities.py b/clients/contract_deployers/communities.py deleted file mode 100644 index 1643cab..0000000 --- a/clients/contract_deployers/communities.py +++ /dev/null @@ -1,15 +0,0 @@ -from clients.foundry import Foundry -from resources.constants import DEPLOYER_ACCOUNT - - -class CommunitiesDeployer: - - def __init__(self, foundry: Foundry): - self.deploy_output = foundry.clone_and_run( - github_org="status-im", - github_repo="communities-contracts", - smart_contract_dir="script", - smart_contract_filename="DeployContracts.s.sol", - private_key=DEPLOYER_ACCOUNT.private_key, - sender_address=DEPLOYER_ACCOUNT.address, - ) diff --git a/clients/contract_deployers/multicall3.py b/clients/contract_deployers/multicall3.py deleted file mode 100644 index d08d871..0000000 --- a/clients/contract_deployers/multicall3.py +++ /dev/null @@ -1,18 +0,0 @@ -from clients.foundry import Foundry - - -class Multicall3Deployer: - - def __init__(self, foundry: Foundry): - multicall3_log = foundry.get_archive("/app/contracts/Multicall3.sol.log") - - with open(multicall3_log, "r") as f: - output = f.read() - for line in output.splitlines(): - if "Deployed to:" in line: - contract_address = line.split("Deployed to:")[1].strip() - print(f"Contract deployed at: {contract_address}") - self.contract_address = contract_address - - if not self.contract_address: - raise Exception("Contract address not found in output.") diff --git a/clients/contract_deployers/snt.py b/clients/contract_deployers/snt.py deleted file mode 100644 index 925d62b..0000000 --- a/clients/contract_deployers/snt.py +++ /dev/null @@ -1,33 +0,0 @@ -from clients.foundry import Foundry -from resources.constants import DEPLOYER_ACCOUNT - -SNTV2_ABI = '[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_amount","type":"uint256"}],"name":"approve","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"creationBlock","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_newController","type":"address"}],"name":"changeController","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_blockNumber","type":"uint256"}],"name":"balanceOfAt","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"version","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_cloneTokenName","type":"string"},{"name":"_cloneDecimalUnits","type":"uint8"},{"name":"_cloneTokenSymbol","type":"string"},{"name":"_snapshotBlock","type":"uint256"},{"name":"_transfersEnabled","type":"bool"}],"name":"createCloneToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"parentToken","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_owner","type":"address"},{"name":"_amount","type":"uint256"}],"name":"generateTokens","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_blockNumber","type":"uint256"}],"name":"totalSupplyAt","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"transfer","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"transfersEnabled","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"parentSnapShotBlock","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_amount","type":"uint256"},{"name":"_extraData","type":"bytes"}],"name":"approveAndCall","outputs":[{"name":"success","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_owner","type":"address"},{"name":"_amount","type":"uint256"}],"name":"destroyTokens","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_token","type":"address"}],"name":"claimTokens","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"tokenFactory","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_transfersEnabled","type":"bool"}],"name":"enableTransfers","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"controller","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"payable":true,"stateMutability":"payable","type":"fallback"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_token","type":"address"},{"indexed":true,"name":"_controller","type":"address"},{"indexed":false,"name":"_amount","type":"uint256"}],"name":"ClaimedTokens","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_from","type":"address"},{"indexed":true,"name":"_to","type":"address"},{"indexed":false,"name":"_amount","type":"uint256"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_cloneToken","type":"address"},{"indexed":false,"name":"_snapshotBlock","type":"uint256"}],"name":"NewCloneToken","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_owner","type":"address"},{"indexed":true,"name":"_spender","type":"address"},{"indexed":false,"name":"_amount","type":"uint256"}],"name":"Approval","type":"event"}]' # noqa: E501 -SNT_TOKEN_CONTROLLER_ABI = '[{"inputs":[{"internalType":"address","name":"owner","type":"address"}],"name":"OwnableInvalidOwner","type":"error"},{"inputs":[{"internalType":"address","name":"account","type":"address"}],"name":"OwnableUnauthorizedAccount","type":"error"},{"inputs":[],"name":"SNTTokenController_NotOpen","type":"error"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"_token","type":"address"},{"indexed":true,"internalType":"address","name":"_controller","type":"address"},{"indexed":false,"internalType":"uint256","name":"_amount","type":"uint256"}],"name":"ClaimedTokens","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"_newController","type":"address"}],"name":"ControllerChanged","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"previousOwner","type":"address"},{"indexed":true,"internalType":"address","name":"newOwner","type":"address"}],"name":"OwnershipTransferStarted","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"previousOwner","type":"address"},{"indexed":true,"internalType":"address","name":"newOwner","type":"address"}],"name":"OwnershipTransferred","type":"event"},{"inputs":[],"name":"acceptOwnership","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address payable","name":"_newController","type":"address"}],"name":"changeController","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"contract MiniMeBase","name":"_token","type":"address"}],"name":"claimTokens","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_from","type":"address"},{"internalType":"uint256","name":"_amount","type":"uint256"}],"name":"destroyTokens","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_to","type":"address"},{"internalType":"uint256","name":"_amount","type":"uint256"}],"name":"generateTokens","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"},{"internalType":"uint256","name":"","type":"uint256"}],"name":"onApprove","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"pure","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"},{"internalType":"uint256","name":"","type":"uint256"}],"name":"onTransfer","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"pure","type":"function"},{"inputs":[],"name":"open","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"owner","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"pendingOwner","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"proxyPayment","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"payable","type":"function"},{"inputs":[],"name":"renounceOwnership","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"bool","name":"_open","type":"bool"}],"name":"setOpen","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"snt","outputs":[{"internalType":"contract MiniMeToken","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"stateMutability":"nonpayable","type":"function"}]' # noqa: E501 - - -class SNTDeployer: - - def __init__(self, foundry: Foundry): - self.deploy_output = foundry.clone_and_run( - github_org="status-im", - github_repo="status-network-token-v2", - smart_contract_dir="script", - smart_contract_filename="Deploy.s.sol", - private_key=DEPLOYER_ACCOUNT.private_key, - sender_address=DEPLOYER_ACCOUNT.address, - ) - - self.snt_contract_address = self._get_snt_contract_address(self.deploy_output) - self.snt_token_controller_address = self._get_snt_token_controller_address(self.deploy_output) - - def _get_snt_contract_address(self, deploy_output): - for deployment in deploy_output.values(): - if deployment["internal_type"] == "contract SNTV2": - return deployment["value"] - raise Exception("SNTV2 contract not found") - - def _get_snt_token_controller_address(self, deploy_output): - for deployment in deploy_output.values(): - if deployment["internal_type"] == "contract SNTTokenController": - return deployment["value"] - raise Exception("SNTTokenController contract not found") diff --git a/clients/expvar.py b/clients/expvar.py deleted file mode 100644 index 16d6540..0000000 --- a/clients/expvar.py +++ /dev/null @@ -1,138 +0,0 @@ -import logging -import threading -import time -from dataclasses import dataclass -from typing import Optional - -import requests - - -@dataclass -class GoMemoryStats: - """Memory statistics from Go's expvar /debug/vars endpoint""" - - alloc_bytes: int # Currently allocated bytes - total_alloc_bytes: int # Total bytes allocated (cumulative) - sys_bytes: int # Bytes obtained from OS - mallocs: int # Number of mallocs - frees: int # Number of frees - heap_alloc_bytes: int # Heap allocated bytes - heap_sys_bytes: int # Heap system bytes - heap_idle_bytes: int # Heap idle bytes - heap_in_use_bytes: int # Heap in-use bytes - heap_released_bytes: int # Heap released bytes - heap_objects: int # Number of heap objects - gc_cpu_fraction: float # GC CPU fraction - num_gc: int # Number of GC runs - - -class ExpvarClient: - """Client for collecting Go memory metrics via /debug/vars endpoint""" - - def __init__(self, base_url: str): - """ - Initialize expvar client - - Args: - base_url: Base URL of the application (e.g., "http://localhost:8080") - """ - self.base_url = base_url.rstrip("/") - self.go_metrics = [] - self._stop_monitoring = None - self.monitor_thread = None - - def get_expvars(self, timeout: int = 10) -> Optional[dict]: - """ - Get memory statistics from /debug/vars endpoint - - Args: - timeout: Request timeout in seconds - - Returns: - MemoryStats object or None if request fails - """ - try: - response = requests.get(f"{self.base_url}/debug/vars", timeout=timeout) - response.raise_for_status() - data = response.json() - data["timestamp"] = time.time() - return data - - except requests.exceptions.RequestException as e: - raise e - except Exception as e: - logging.error(f"Error parsing /debug/vars response: {e}") - return None - - @staticmethod - def parse_expvars(data): - memstats = data.get("memstats", {}) - if not memstats: - raise ValueError("memstats not found in /debug/vars response") - - return GoMemoryStats( - alloc_bytes=memstats.get("Alloc", 0), - total_alloc_bytes=memstats.get("TotalAlloc", 0), - sys_bytes=memstats.get("Sys", 0), - mallocs=memstats.get("Mallocs", 0), - frees=memstats.get("Frees", 0), - heap_alloc_bytes=memstats.get("HeapAlloc", 0), - heap_sys_bytes=memstats.get("HeapSys", 0), - heap_idle_bytes=memstats.get("HeapIdle", 0), - heap_in_use_bytes=memstats.get("HeapInuse", 0), - heap_released_bytes=memstats.get("HeapReleased", 0), - heap_objects=memstats.get("HeapObjects", 0), - gc_cpu_fraction=memstats.get("GCCPUFraction", 0.0), - num_gc=memstats.get("NumGC", 0), - ) - - @staticmethod - def parse_num_goroutines(data): - return data.get("numGoroutine", 0) - - @staticmethod - def parse_num_threads(data): - return data.get("numThreads", 0) - - @staticmethod - def parse_timestamp(data): - return data.get("timestamp", 0) - - def start_monitoring(self, interval: float = 1.0): - """Start independent Go metrics monitoring thread - - Args: - interval: Monitoring interval in seconds - """ - self.go_metrics = [] - self._stop_monitoring = threading.Event() - - def monitor_go_metrics(): - while self._stop_monitoring and not self._stop_monitoring.is_set(): - try: - expvars = self.get_expvars() - if expvars: - self.go_metrics.append(expvars) - - # Wait for the specified interval or until stop event is set - self._stop_monitoring.wait(timeout=interval) - except Exception as e: - logging.error(f"Go metrics monitoring error: {e}") - self._stop_monitoring.set() - - self._stop_monitoring.clear() - self.monitor_thread = threading.Thread(target=monitor_go_metrics, daemon=True) - self.monitor_thread.start() - logging.info("Started Go metrics monitoring") - - def stop_monitoring(self): - """Stop the Go metrics monitoring thread and return collected metrics""" - if self._stop_monitoring: - self._stop_monitoring.set() - - if self.monitor_thread and self.monitor_thread.is_alive(): - self.monitor_thread.join(timeout=10) - if self.monitor_thread.is_alive(): - logging.warning("Go metrics monitoring thread didn't stop gracefully") - - return self.go_metrics diff --git a/clients/foundry.py b/clients/foundry.py deleted file mode 100644 index 9d1c7b9..0000000 --- a/clients/foundry.py +++ /dev/null @@ -1,190 +0,0 @@ -import io -import json -import logging -import tarfile -import tempfile -import time -import docker -import docker.errors -import os - -from utils.config import Config -from resources.constants import user_1, ANVIL_NETWORK_ID -from tenacity import retry, wait_fixed, stop_after_attempt - - -class Foundry: - - container = None - - def __init__(self): - self.docker_client = docker.from_env() - self.docker_project_name = Config.docker_project_name - self.network_name = f"{self.docker_project_name}_default" - - container_name_prefix = f"{self.docker_project_name}-foundry" - self.container_name = self.find_container_name(self.network_name, container_name_prefix) - - if not self.container_name: - raise Exception("Foundry container not found") - self.container = self.docker_client.containers.get(self.container_name) - self.wait_for_healthy() - - @retry(stop=stop_after_attempt(10), wait=wait_fixed(0.1), reraise=True) - def find_container_name(self, network_name, searched_container): - network = self.docker_client.networks.get(network_name) - - for container in network.containers: - container_name = container.name - if container_name is not None and searched_container in container_name: - return container_name - - return None - - def wait_for_healthy(self, timeout=10): - start_time = time.time() - while time.time() - start_time <= timeout: - if self.is_connected(): - logging.info(f"Foundry is healthy after {time.time() - start_time} seconds") - return - else: - time.sleep(0.1) - raise TimeoutError(f"Foundry was not healthy after {timeout} seconds") - - def is_connected(self): - if not self.container: - return False - - exec_result = self.container.exec_run("cast chain-id") - exit_code = exec_result.exit_code - if exit_code != 0: - logging.info(f"Exit code: {exit_code}") - return False - output = exec_result.output.decode().strip() - if output != str(ANVIL_NETWORK_ID): - logging.info(f"ChainID comparison error. Expected: {output}, Actual:{ANVIL_NETWORK_ID}") - return False - return True - - def clone_and_run(self, **kwargs): - if not self.container: - raise Exception("Container not found") - - github_org = kwargs.get("github_org", "status-im") - github_repo = kwargs.get("github_repo") - if not github_repo: - raise ValueError("github_repo is required") - smart_contract_dir = kwargs.get("smart_contract_dir") - if not smart_contract_dir: - raise ValueError("smart_contract_dir is required") - smart_contract_filename = kwargs.get("smart_contract_filename") - if not smart_contract_filename: - raise ValueError("smart_contract_filename is required") - private_key = kwargs.get("private_key", user_1.private_key) - sender_address = kwargs.get("sender_address", user_1.address) - - cmd = f"/app/clone_and_run.sh {github_org} {github_repo} {smart_contract_dir} {smart_contract_filename} {private_key} {sender_address}" - logging.info(f"Running command: {cmd}") - - exec_result = self.container.exec_run( - f"{cmd}", - workdir="/app", - ) - logging.info(f"Exit code: {exec_result.exit_code}") - logging.info(f"Result: {exec_result.output.decode().strip()}") - if exec_result.exit_code != 0: - raise Exception(f"Failed to clone and run {github_repo}") - - container_output_path = f"/app/{github_repo}/broadcast/{smart_contract_filename}/{ANVIL_NETWORK_ID}/run-latest.json" - host_output_path = self.get_archive(container_output_path) - if not host_output_path: - raise Exception(f"Failed to extract data from {container_output_path}") - with open(host_output_path, "r") as f: - output = json.load(f) - return output["returns"] - - def put_and_deploy(self, data, contract_path, contract_name, **kwargs): - if not self.container: - raise Exception("Container not found") - - container_path = self.put_archive(data, **kwargs) - - private_key = kwargs.get("private_key", user_1.private_key) - sender_address = kwargs.get("sender_address", user_1.address) - - cmd = f"""forge create {container_path}/{contract_path}:{contract_name} - --rpc-url 'http://anvil:8545' - --from {sender_address} - --private-key {private_key} - --broadcast""" - constructor_args = kwargs.get("constructor_args") - if constructor_args: - cmd += f" --constructor-args {constructor_args}" - - logging.info(f"Running command: {cmd}") - exec_result = self.container.exec_run( - f"{cmd}", - workdir="/app", - ) - exit_code = exec_result.exit_code - output = exec_result.output.decode() - - logging.info(f"Exit code: {exit_code}") - logging.info(f"Result: {output.strip()}") - if exit_code != 0: - raise Exception(f"Failed to deploy {contract_name}") - - # Extract contract address from output - for line in output.splitlines(): - if "Deployed to:" in line: - contract_address = line.split("Deployed to:")[1].strip() - print(f"Contract deployed at: {contract_address}") - return contract_address - raise Exception("Contract address not found in output.") - - def put_archive(self, data, **kwargs): - if not self.container: - raise Exception("Container not found") - - container_path = kwargs.get("container_path") - if not container_path: - # Create a temporary directory - temp_dir_name = tempfile.mktemp(prefix="temp_", dir="/app").split("/")[-1] - temp_dir_path = f"/app/{temp_dir_name}" # Directory path inside the container - - # Create the temporary directory in the container - create_dir_cmd = f"mkdir -p {temp_dir_path}" - exec_response = self.container.exec_run(create_dir_cmd) - if exec_response.exit_code != 0: - raise Exception(f"Failed to create directory: {exec_response.output.decode()}") - - container_path = temp_dir_path - logging.info(f"Putting archive in path: {container_path}") - - try: - self.container.put_archive(container_path, data) - except docker.errors.NotFound: - raise Exception(f"Path '{container_path}' not found in container {self.container.name}") - - return container_path - - def get_archive(self, container_path): - if not self.container: - raise Exception("Container not found") - - try: - stream, _ = self.container.get_archive(container_path) - except docker.errors.NotFound: - raise Exception(f"Path '{container_path}' not found in container {self.container.name}") - - temp_dir = tempfile.mkdtemp() - tar_bytes = io.BytesIO(b"".join(stream)) - - with tarfile.open(fileobj=tar_bytes) as tar: - tar.extractall(path=temp_dir) - # If the tar contains a single file, return the path to that file - # Otherwise it's a directory, just return temp_dir. - if len(tar.getmembers()) == 1: - return os.path.join(temp_dir, tar.getmembers()[0].name) - - return temp_dir diff --git a/clients/gorush_stub.py b/clients/gorush_stub.py deleted file mode 100644 index 347121b..0000000 --- a/clients/gorush_stub.py +++ /dev/null @@ -1,160 +0,0 @@ -import json -import logging -import threading -import time -from http.server import BaseHTTPRequestHandler, HTTPServer - -import requests - - -class GorushRequestHandler(BaseHTTPRequestHandler): - """HTTP request handler for gorush stub""" - - # Class-level variable to store requests for debugging - push_requests = [] - - def log_message(self, format, *args): - logging.debug(f"gorush stub request: {format % args}") - - def _set_response(self, status_code=200, content_type="application/json"): - self.send_response(status_code) - self.send_header("Content-type", content_type) - self.end_headers() - - def do_GET(self): - if self.path == "/healthz": - self._set_response() - self.wfile.write(json.dumps({"status": "ok"}).encode("utf-8")) - else: - self._set_response(404) - self.wfile.write(json.dumps({"error": "Not found"}).encode("utf-8")) - - def do_POST(self): - content_length = int(self.headers["Content-Length"]) - post_data = self.rfile.read(content_length).decode("utf-8") - - if self.path == "/api/push": - # Store request for debugging - self.__class__.push_requests.append(post_data) - - # Decode post_data - try: - request = json.loads(post_data) - except json.JSONDecodeError: - self._set_response(400) - self.wfile.write(json.dumps({"error": "Invalid JSON"}).encode("utf-8")) - return - - # Return a successful response - self._set_response() - response = { - "success": "ok", - "counts": { - "total": len(request["notifications"]), - }, - } - self.wfile.write(json.dumps(response).encode("utf-8")) - else: - self._set_response(404) - self.wfile.write(json.dumps({"error": "Not found"}).encode("utf-8")) - - -class GorushStub: - """Client for interacting with gorush-stub service""" - - def __init__(self, address="localhost", port=8088): - """Initialize GorushStub client - - Args: - port: Port to expose gorush-stub on the host - """ - # Setup logging - self.logger = logging.getLogger(__name__) - - # Initialize the HTTP server - self.base_url = "" - self.server = None - self.server_thread = None - - # Clear previous debug requests - GorushRequestHandler.push_requests = [] - - # Create and start the HTTP server - self.server = HTTPServer((address, port), GorushRequestHandler) - self.server_thread = threading.Thread(target=self.server.serve_forever) - self.server_thread.daemon = True - self.server_thread.start() - - # Create base URL for API requests - self.base_url = f"http://{address}:{self.server.server_port}" - - # Wait for the server to start - self._wait_for_service() - self.logger.info(f"gorush-stub initialized at {self.base_url}") - - def _wait_for_service(self, timeout=30, interval=1): - """Wait for gorush-stub service to be available - - Args: - timeout: Maximum time to wait in seconds - interval: Interval between attempts in seconds - - Returns: - bool: True if service is available, False otherwise - """ - start_time = time.time() - while time.time() - start_time < timeout: - try: - response = requests.get(f"{self.base_url}/healthz") - if response.status_code == 200: - return True - except requests.RequestException: - pass - time.sleep(interval) - - self.logger.error(f"gorush-stub service not available after {timeout} seconds") - return False - - def get_requests(self): - """Get debug requests from gorush-stub - - Returns: - list: List of recorded requests - """ - # Get a copy of the current push_requests - requests = GorushRequestHandler.push_requests.copy() - - # Clear the original list - GorushRequestHandler.push_requests = [] - return requests - - def wait_for_requests(self, timeout=10): - start_time = time.time() - push_requests = [] - - while len(push_requests) == 0: - if time.time() - start_time > timeout: - assert False, "Timeout waiting for push notifications requests" - - time.sleep(1) - - for req in self.get_requests(): - for notification in json.loads(req)["notifications"]: - push_requests.append(notification) - - return push_requests - - def close(self): - """Stop the gorush-stub HTTP server""" - if not self.server: - return True - - try: - self.server.shutdown() - self.server.server_close() - self.server = None - self.server_thread = None - return True - except Exception as e: - self.logger.error(f"Failed to close gorush-stub server: {e}") - return False diff --git a/clients/metrics.py b/clients/metrics.py deleted file mode 100644 index c2333c8..0000000 --- a/clients/metrics.py +++ /dev/null @@ -1,570 +0,0 @@ -import json -import logging -import os -import statistics -import time -from dataclasses import dataclass - -import matplotlib -import matplotlib.pyplot as plt - -from clients.expvar import ExpvarClient - -matplotlib.use("Agg") # Use non-interactive backend -logging.getLogger("matplotlib.font_manager").setLevel(logging.WARNING) - - -@dataclass -class CPUMetrics: - cpu_percent: float - cpu_count: int - - -@dataclass -class RAMMetrics: - memory_usage_mb: float - memory_max_usage_mb: float - - -@dataclass -class NetworkMetrics: - rx_bytes: int # Received bytes - tx_bytes: int # Transmitted bytes - rx_packets: int # Received packets - tx_packets: int # Transmitted packets - rx_dropped: int # Received packets dropped - tx_dropped: int # Transmitted packets dropped - rx_errors: int # Receive errors - tx_errors: int # Transmit errors - rx_bytes_per_sec: float = 0 # Bytes per second received - tx_bytes_per_sec: float = 0 # Bytes per second transmitted - - -@dataclass -class GoMemStats: - idle_memory_mb: float # Heap idle memory in MB - heap_alloc_mb: float # Currently allocated heap memory in MB - heap_sys_mb: float # Heap system memory in MB - heap_in_use_mb: float # Heap in-use memory in MB - num_gc: int # Number of GC runs - gc_cpu_fraction: float # GC CPU fraction - - -class Events: - def __init__(self): - self.events = {} - - def append(self, event: str): - logging.info(f"Metrics event: {event}") - self.events[event] = time.time() - - def __iter__(self): - return iter(self.events) - - def to_dict(self): - return self.events - - -def calculate_cpu_metrics(stats): - # CPU Usage fields - cpu_stats = stats["cpu_stats"] - precpu_stats = stats["precpu_stats"] - - # Total CPU usage in nanoseconds - cpu_total = cpu_stats["cpu_usage"]["total_usage"] - cpu_total_prev = precpu_stats["cpu_usage"]["total_usage"] - - # System CPU usage in nanoseconds - system_total = cpu_stats.get("system_cpu_usage", 0) - system_total_prev = precpu_stats.get("system_cpu_usage", 0) - - # CPU cores - try: - try: - cpu_count = len(cpu_stats["cpu_usage"]["percpu_usage"]) - except KeyError: - cpu_count = cpu_stats["online_cpus"] - except KeyError: - cpu_count = 1 - - # Calculate deltas - cpu_delta = cpu_total - cpu_total_prev - system_delta = system_total - system_total_prev - - # Calculate percentages - cpu_percent = 0.0 - if system_delta > 0 and cpu_delta > 0: - cpu_percent = (cpu_delta / system_delta) * cpu_count * 100.0 - - return CPUMetrics( - cpu_percent=cpu_percent, - cpu_count=cpu_count, - ) - - -def calculate_memory_metrics(stats): - usage = stats["memory_stats"]["usage"] - max_usage = stats["memory_stats"].get("max_usage", usage) # Use current usage as fallback - - # Convert to MB for readability - mb = 1024 * 1024 - return RAMMetrics( - memory_usage_mb=usage / mb, - memory_max_usage_mb=max_usage / mb, - ) - - -def calculate_network_metrics(stats, prev_stats=None): - """Calculate network metrics from Docker stats - - Args: - stats: Current Docker stats containing network information - prev_stats: Previous stats for calculating rates (optional) - - Returns: - NetworkMetrics: Network statistics - """ - network_stats = stats.get("networks", {}) - - # Initialize totals - total_rx_bytes = 0 - total_tx_bytes = 0 - total_rx_packets = 0 - total_tx_packets = 0 - total_rx_dropped = 0 - total_tx_dropped = 0 - total_rx_errors = 0 - total_tx_errors = 0 - - # Sum up all network interfaces - for interface_name, interface_stats in network_stats.items(): - total_rx_bytes += interface_stats.get("rx_bytes", 0) - total_tx_bytes += interface_stats.get("tx_bytes", 0) - total_rx_packets += interface_stats.get("rx_packets", 0) - total_tx_packets += interface_stats.get("tx_packets", 0) - total_rx_dropped += interface_stats.get("rx_dropped", 0) - total_tx_dropped += interface_stats.get("tx_dropped", 0) - total_rx_errors += interface_stats.get("rx_errors", 0) - total_tx_errors += interface_stats.get("tx_errors", 0) - - # Calculate rates if previous stats are available - rx_bytes_per_sec = 0 - tx_bytes_per_sec = 0 - - if prev_stats is not None: - prev_network_stats = prev_stats.get("networks", {}) - prev_rx_bytes = 0 - prev_tx_bytes = 0 - - for interface_name, interface_stats in prev_network_stats.items(): - prev_rx_bytes += interface_stats.get("rx_bytes", 0) - prev_tx_bytes += interface_stats.get("tx_bytes", 0) - - # Calculate time difference - current_time = stats.get("read", "") - prev_time = prev_stats.get("read", "") - - if current_time and prev_time: - try: - from datetime import datetime - - current_dt = datetime.fromisoformat(current_time.replace("Z", "+00:00")) - prev_dt = datetime.fromisoformat(prev_time.replace("Z", "+00:00")) - time_diff = (current_dt - prev_dt).total_seconds() - except (ValueError, AttributeError): - # If timestamp parsing fails, use a default time difference - time_diff = 1.0 - else: - # Fallback: assume 1 second interval if no timestamps - time_diff = 1.0 - - # Calculate rates - if time_diff > 0: - rx_bytes_per_sec = (total_rx_bytes - prev_rx_bytes) / time_diff - tx_bytes_per_sec = (total_tx_bytes - prev_tx_bytes) / time_diff - else: - # If time_diff is 0 or negative, use the difference as bytes per second - rx_bytes_per_sec = total_rx_bytes - prev_rx_bytes - tx_bytes_per_sec = total_tx_bytes - prev_tx_bytes - - return NetworkMetrics( - rx_bytes=total_rx_bytes, - tx_bytes=total_tx_bytes, - rx_packets=total_rx_packets, - tx_packets=total_tx_packets, - rx_dropped=total_rx_dropped, - tx_dropped=total_tx_dropped, - rx_errors=total_rx_errors, - tx_errors=total_tx_errors, - rx_bytes_per_sec=max(0, rx_bytes_per_sec), # Ensure non-negative - tx_bytes_per_sec=max(0, tx_bytes_per_sec), # Ensure non-negative - ) - - -@dataclass -class ContainerStats: - """Container stats object""" - - def __init__(self, stat, prev_stat=None, go_memory_stats=None): - self.timestamp = time.time() - self.cpu = calculate_cpu_metrics(stat) - self.ram = calculate_memory_metrics(stat) - self.network = calculate_network_metrics(stat, prev_stat) - self.expvars = go_memory_stats - - -@dataclass -class StatusGoMetrics: - # Container for performance monitoring metrics - duration = 0 - samples = 0 - cpu_median = 0 - cpu_max = 0 - ram_median = 0 - ram_max = 0 - rx_bytes_per_sec_median = 0 - rx_bytes_per_sec_max = 0 - rx_total_bytes = 0 - ex_total_packets = 0 - tx_bytes_per_sec_median = 0 - tx_bytes_per_sec_max = 0 - tx_total_bytes = 0 - tx_total_packets = 0 - total_network_errors = 0 - - # Expvars metrics - total_memory_median = 0 - total_memory_max = 0 - idle_memory_median = 0 # "Idle" that is kepy by Go and not released to OS - idle_memory_max = 0 - final_gc_count = 0 - timestamp = 0 - version = "" - - def __init__( - self, - container_stats: list[ContainerStats] | None = None, - go_metrics: list[dict] | None = None, - events: Events | None = None, - version: str = "", - stats: list[ContainerStats] | None = None, - ): - """ - Initialize PerformanceMetrics with independent arrays - - Args: - container_stats: List of container statistics with their own timestamps - go_metrics: List of Go memory statistics with their own timestamps - events: Events tracker - version: Version string - stats: Legacy parameter for backward compatibility - """ - # Handle backward compatibility - if stats is not None and container_stats is None: - container_stats = stats - - self.container_stats = container_stats or [] - self.go_metrics = go_metrics or [] - self._memory_stats = [ExpvarClient.parse_expvars(metric) for metric in self.go_metrics] - self.events = events or Events() - self.timestamp = time.time() - self.version = version - - self._calculate_metrics() - - def _calculate_container_metrics(self): - # Calculate duration from container stats - self.duration = self.container_stats[-1].timestamp - self.container_stats[0].timestamp - - # Extract CPU and RAM metrics - cpu_percents = [stat.cpu.cpu_percent for stat in self.container_stats] - ram_usage = [stat.ram.memory_usage_mb for stat in self.container_stats] - - # Extract network metrics - rx_bytes_per_sec = [stat.network.rx_bytes_per_sec for stat in self.container_stats] - tx_bytes_per_sec = [stat.network.tx_bytes_per_sec for stat in self.container_stats] - - self.samples = len(self.container_stats) - self.cpu_median = statistics.median(cpu_percents) - self.cpu_max = max(cpu_percents) - self.ram_median = statistics.median(ram_usage) - self.ram_max = max(ram_usage) - - # Network metrics - self.rx_bytes_per_sec_median = statistics.median(rx_bytes_per_sec) - self.rx_bytes_per_sec_max = max(rx_bytes_per_sec) - self.tx_bytes_per_sec_median = statistics.median(tx_bytes_per_sec) - self.tx_bytes_per_sec_max = max(tx_bytes_per_sec) - - # Total network statistics from the last sample - last_stat = self.container_stats[-1] - self.rx_total_bytes = last_stat.network.rx_bytes - self.tx_total_bytes = last_stat.network.tx_bytes - self.ex_total_packets = last_stat.network.rx_packets - self.tx_total_packets = last_stat.network.tx_packets - self.total_network_errors = ( - last_stat.network.rx_errors + last_stat.network.tx_errors + last_stat.network.rx_dropped + last_stat.network.tx_dropped - ) - - def _calculate_memory_stats(self): - """Calculate memory statistics from collected go metrics""" - - # Convert to MB for consistency - mb = 1024 * 1024 - - total_memory = [metric.sys_bytes - metric.heap_released_bytes for metric in self._memory_stats] - idle_memory = [metric.heap_idle_bytes - metric.heap_released_bytes for metric in self._memory_stats] - - if total_memory: - self.total_memory_median = statistics.median(total_memory) / mb - self.total_memory_max = max(total_memory) / mb - if idle_memory: - self.idle_memory_median = statistics.median(idle_memory) / mb - self.idle_memory_max = max(idle_memory) / mb - - # Final GC count from the last sample - self.final_gc_count = self._memory_stats[-1].num_gc - - def _calculate_go_metrics(self): - """Calculate summary metrics from collected Go memory data""" - self._calculate_memory_stats() - self._num_goroutines = [ExpvarClient.parse_num_goroutines(metric) for metric in self.go_metrics] - self.num_goroutines_max = max(self._num_goroutines) - self._num_threads = [ExpvarClient.parse_num_threads(metric) for metric in self.go_metrics] - self.num_threads_max = max(self._num_threads) - - def _calculate_metrics(self): - """Calculate summary metrics from collected data""" - if self.container_stats: - self._calculate_container_metrics() - if self.go_metrics: - self._calculate_go_metrics() - - def to_dict(self): - """Convert PerformanceMetrics to a JSON-serializable dictionary""" - return { - "timestamp": self.timestamp, - "version": self.version, - "events": self.events.to_dict(), - "metrics": { - "cpu": { - "median": self.cpu_median, - "max": self.cpu_max, - }, - "ram": { - "median": self.ram_median, - "max": self.ram_max, - }, - "network": { - "rx": { - "bytes_per_sec": { - "median": self.rx_bytes_per_sec_median, - "max": self.rx_bytes_per_sec_max, - }, - "total_bytes": self.rx_total_bytes, - "total_packets": self.ex_total_packets, - }, - "tx": { - "bytes_per_sec": { - "median": self.tx_bytes_per_sec_median, - "max": self.tx_bytes_per_sec_max, - }, - "total_bytes": self.tx_total_bytes, - "total_packets": self.tx_total_packets, - }, - "total_errors": self.total_network_errors, - }, - "expvar": { - "idle_memory_mb": { - "median": self.idle_memory_median, - "max": self.idle_memory_max, - }, - "total_memory_mb": { - "median": self.total_memory_median, - "max": self.total_memory_max, - }, - "gc_count": self.final_gc_count, - "num_goroutines_max": self.num_goroutines_max, - "num_threads_max": self.num_threads_max, - }, - }, - } - - def save_performance_chart(self, title: str, output_path=None): - """Generate and save a performance chart as a PNG image - - Args: - title: Chart title - output_path: Path to save the chart. If None, saves to ./performance_metrics_{container_id}.png - - Returns: - str: Path to the saved chart file - """ - if not self.container_stats or not self.go_metrics: - raise ValueError("No performance data to generate chart") - - mb = 1024 * 1024 - - # Create a figure with four subplots (CPU, Memory, Network, and Accumulated Network) - fig, (ax1, ax2, ax3, ax4, ax5, ax6) = plt.subplots(6, 1, figsize=(12, 18), sharex=True) - fig.suptitle(title, fontsize=16, y=0.98) - - # Extract data from container stats - container_timestamps = [stat.timestamp for stat in self.container_stats] - cpu_values = [stat.cpu.cpu_percent for stat in self.container_stats] - ram_values = [stat.ram.memory_usage_mb for stat in self.container_stats] - rx_values = [stat.network.rx_bytes_per_sec / mb for stat in self.container_stats] - tx_values = [stat.network.tx_bytes_per_sec / mb for stat in self.container_stats] - - # Convert to relative time - start_time = container_timestamps[0] - container_time_points = [t - start_time for t in container_timestamps] - - # Extract accumulated network data - rx_bytes = [stat.network.rx_bytes for stat in self.container_stats] - tx_bytes = [stat.network.tx_bytes for stat in self.container_stats] - rx_bytes_mb = [bytes / mb for bytes in rx_bytes] - tx_bytes_mb = [bytes / mb for bytes in tx_bytes] - - # Extract data from Go metrics independently - go_timestamps = [ExpvarClient.parse_timestamp(metric) for metric in self.go_metrics] - sys_values = [metric.sys_bytes / mb for metric in self._memory_stats] - actual_memory_values = [(metric.sys_bytes - metric.heap_released_bytes) / mb for metric in self._memory_stats] - could_be_released = [(metric.heap_idle_bytes - metric.heap_released_bytes) / mb for metric in self._memory_stats] - - # Convert to relative time (use container start time if available, otherwise Go metrics start time) - go_start_time = start_time if self.container_stats else go_timestamps[0] - go_time_points = [t - go_start_time for t in go_timestamps] - - # CPU usage plot - cpu_median = statistics.median(cpu_values) - cpu_max = max(cpu_values) - ax1.plot(container_time_points, cpu_values, "b-", label="CPU Usage (%)") - ax1.set_ylabel("CPU Usage (%)") - ax1.set_title("CPU Usage Over Time") - ax1.grid(True) - ax1.set_xlim(left=0) - ax1.set_ylim(bottom=0) - ax1.legend(loc="best") - - # Memory usage plot with independent arrays - median_memory = statistics.median(ram_values) - max_memory = max(ram_values) - ax2.plot(container_time_points, ram_values, "m-", label="Container Memory (MB)") - - sys_median = statistics.median(sys_values) - sys_max = max(sys_values) - ax2.plot(go_time_points, sys_values, "orange", label="Go Sys Memory (MB)", linewidth=2) - - actual_memory_median = statistics.median(actual_memory_values) - actual_memory_max = max(actual_memory_values) - ax2.plot(go_time_points, actual_memory_values, "g-", label="Go Actual Memory Usage (MB)", linewidth=2) - ax2.plot(go_time_points, could_be_released, "b", label="Go Idle Memory (MB)", linewidth=2) - - ax2.set_ylabel("Memory Usage (MB)") - ax2.set_title("Memory Usage Over Time") - ax2.grid(True) - ax2.set_xlim(left=0) - ax2.set_ylim(bottom=0) - ax2.legend(loc="best") - - # Network usage plot - rx_median = statistics.median(rx_values) - tx_median = statistics.median(tx_values) - rx_max = max(rx_values) - tx_max = max(tx_values) - ax3.plot(container_time_points, rx_values, "c-", label="Download (MB/s)", linewidth=2) - ax3.plot(container_time_points, tx_values, "r-", label="Upload (MB/s)", linewidth=2) - ax3.set_ylabel("Network Throughput (MB/s)") - ax3.set_title("Network Activity Over Time") - ax3.grid(True) - ax3.set_xlim(left=0) - ax3.set_ylim(bottom=0) - ax3.legend(loc="best", labelspacing=2) - - # Accumulated network usage plot - rx_total_bytes = rx_bytes[-1] - tx_total_bytes = tx_bytes[-1] - ax4.plot(container_time_points, rx_bytes_mb, "c-", label=f"Download (MB), total: {rx_total_bytes / mb:.2f} MB", linewidth=2) - ax4.plot(container_time_points, tx_bytes_mb, "r-", label=f"Upload (MB), total: {tx_total_bytes / mb:.2f} MB", linewidth=2) - ax4.set_xlabel("Time (seconds)") - ax4.set_ylabel("Total Data Transferred (MB)") - ax4.set_title("Accumulated Network Data Over Time") - ax4.grid(True) - ax4.set_xlim(left=0) - ax4.set_ylim(bottom=0) - ax4.legend(loc="best") - - # Number of goroutines plot - ax5.plot(go_time_points, self._num_goroutines, "g-", label="Number of Goroutines", linewidth=2) - ax5.plot(go_time_points, self._num_threads, "b-", label="Number of Threads", linewidth=2) - ax5.set_xlabel("Time (seconds)") - ax5.set_ylabel("Numbers") - ax5.set_title("Various Numbers Over Time") - ax5.grid(True) - ax5.set_xlim(left=0) - ax5.set_ylim(bottom=0) - ax5.legend(loc="best") - - # Add vertical lines for events across all plots - if self.events and hasattr(self.events, "events") and self.events.events: - for event_name, event_timestamp in self.events.events.items(): - # Convert the event timestamp to relative time (seconds from start) - event_time = event_timestamp - start_time - - # Only add lines for events that occur within our time range - if container_time_points and 0 <= event_time <= max(container_time_points): - # Add vertical line to all subplots - for ax in [ax1, ax2, ax3, ax4, ax5]: - ax.axvline(x=event_time, color="black", linestyle="--", alpha=0.7, linewidth=1) - - # Add an event label to the top plot (CPU) to avoid cluttering - ax1.text( - event_time, - ax1.get_ylim()[1] * 0.95, - event_name, - rotation=90, - verticalalignment="top", - horizontalalignment="right", - fontsize=8, - color="black", - alpha=0.8, - ) - - # Create consolidated statistical summary outside plots - stats_text = "Performance Statistics:\n" - stats_text += f"- CPU Usage: median = {cpu_median:.2f}%, max = {cpu_max:.2f}%\n" - stats_text += f"- Container Memory: median = {median_memory:.2f} MB, max = {max_memory:.2f} MB\n" - stats_text += f"- Go Sys Memory: median = {sys_median:.2f} MB, max = {sys_max:.2f} MB\n" - stats_text += f"- Go Actual Memory: median = {actual_memory_median:.2f} MB, max = {actual_memory_max:.2f} MB\n" - stats_text += f"- Network Download: median = {rx_median:.2f} MB/s, max = {rx_max:.2f} MB/s\n" - stats_text += f"- Network Upload: median = {tx_median:.2f} MB/s, max = {tx_max:.2f} MB/s" - - # Adjust layout to make room for the statistics text at the bottom - plt.tight_layout(rect=(0, 0.15, 1, 1)) - - ax6.axis("off") - ax6.invert_yaxis() - ax6.text(0.5, 0.5, stats_text, verticalalignment="top") - - # Save the figure - if output_path is None: - timestamp = time.strftime("%Y%m%d-%H%M%S") - output_path = f"./performance_metrics_{timestamp}.png" - - # Ensure directory exists - os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) - - # Save figure - plt.savefig(output_path, dpi=100, bbox_inches="tight") - plt.close(fig) - - logging.info(f"Performance chart saved to {output_path}") - return output_path - - def save_to_file(self, filename: str): - metrics = self.to_dict() - os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, "w") as f: - json.dump(metrics, f, indent=2) - logging.info(f"Performance report saved to {filename}") diff --git a/clients/push_notification_server.py b/clients/push_notification_server.py deleted file mode 100644 index 2703028..0000000 --- a/clients/push_notification_server.py +++ /dev/null @@ -1,17 +0,0 @@ -from clients.statusgo_container import PushNotificationServerContainer - - -class PushNotificationServer: - container = None - - def __init__(self, gorush_port=8080): - self.gorush_port = gorush_port - self.container = PushNotificationServerContainer( - identity="3e64442a0ba8a59b4d2dc7385cd4533a10e86dd644e7ec549cb92503787f5282", - gorush_port=self.gorush_port, - ) - - self.data_dir = self.container.data_dir() - self.container.start_health_monitoring() - - assert self.data_dir != "" diff --git a/clients/rpc.py b/clients/rpc.py deleted file mode 100644 index cbb1281..0000000 --- a/clients/rpc.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -from clients.api import ApiClient - - -class RpcClient(ApiClient): - - def __init__(self, client=requests.Session()): - self.client = client - self._request_id = 0 - - @property - def request_id(self) -> int: - self._request_id += 1 - return self._request_id - - def validate_json_rpc_response(self, response, _id): - # Must contain exactly one of 'result' or 'error' - has_result = "result" in response - has_error = "error" in response - - if not (has_result ^ has_error): # True only if exactly one is True - raise AssertionError(f"Invalid structure: must contain exactly one of 'result' or 'error', got: {response}") - - try: - if _id != response["id"]: - raise AssertionError(f"got id: {response['id']} instead of expected id: {_id}") - except KeyError: - raise AssertionError(f"no id in response {response}") - - return response - - def rpc_valid_request(self, method, params=None): - request_id = self.request_id - - if params is None: - params = [] - data = {"jsonrpc": "2.0", "method": method, "id": request_id} - if params: - data["params"] = params - response = self.api_request_json("CallRPC", data) - self.validate_json_rpc_response(response, request_id) - return response.get("result") diff --git a/clients/services/accounts.py b/clients/services/accounts.py deleted file mode 100644 index c81d916..0000000 --- a/clients/services/accounts.py +++ /dev/null @@ -1,219 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service -from utils import fake - - -class AccountService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "accounts") - - def get_accounts(self): - response = self.rpc_request("getAccounts") - return response - - def get_account_keypairs(self): - response = self.rpc_request("getKeypairs") - return response - - def add_account(self, password, account_data): - params = [password, account_data] - response = self.rpc_request("addAccount", params) - return response - - def add_watch_only_account(self, address: str, name: str, color: str = "blue"): - account_data = { - "address": address, - "key-uid": "", - "wallet": False, - "chat": False, - "type": "watch", - "path": "", - "public-key": "", - "name": name, - "emoji": fake.emoji(), - "colorId": color, - } - params = ["", account_data] - response = self.rpc_request("addAccount", params) - return response - - def delete_account(self, account_address, password): - params = [account_address, password] - response = self.rpc_request("deleteAccount", params) - return response - - def import_mnemonic(self, mnemonic, password): - params = [mnemonic, password] - response = self.rpc_request("importMnemonic", params) - return response - - def add_keypair_via_seed_phrase(self, mnemonic, password, name, wallet_account): - params = [mnemonic, password, name, wallet_account] - response = self.rpc_request("addKeypairViaSeedPhrase", params) - return response - - def add_keypair_via_private_key(self, private_key, password, name, wallet_account): - params = [private_key, password, name, wallet_account] - response = self.rpc_request("addKeypairViaPrivateKey", params) - return response - - def verify_password(self, password): - params = [password] - response = self.rpc_request("verifyPassword", params) - return response - - def resolve_suggested_path_for_keypair(self, key_uid): - params = [key_uid] - response = self.rpc_request("resolveSuggestedPathForKeypair", params) - return response - - def has_paired_devices(self): - response = self.rpc_request("hasPairedDevices", []) - return response - - def update_keypair_name(self, key_uid, name): - params = [key_uid, name] - response = self.rpc_request("updateKeypairName", params) - return response - - def move_wallet_account(self, from_position, to_position): - params = [from_position, to_position] - response = self.rpc_request("moveWalletAccount", params) - return response - - def update_token_preferences(self, preferences): - params = [preferences] - response = self.rpc_request("updateTokenPreferences", params) - return response - - def get_token_preferences(self): - response = self.rpc_request("getTokenPreferences", []) - return response - - def update_collectible_preferences(self, preferences): - params = [preferences] - response = self.rpc_request("updateCollectiblePreferences", params) - return response - - def get_collectible_preferences(self): - response = self.rpc_request("getCollectiblePreferences", []) - return response - - def get_account_by_address(self, address): - params = [address] - response = self.rpc_request("getAccountByAddress", params) - return response - - def get_keypair_by_key_uid(self, key_uid): - params = [key_uid] - response = self.rpc_request("getKeypairByKeyUID", params) - return response - - def update_account(self, account): - params = [account] - response = self.rpc_request("updateAccount", params) - return response - - def save_or_update_keycard(self, keycard, password): - params = [keycard, password] - response = self.rpc_request("saveOrUpdateKeycard", params) - return response - - def delete_keycard(self, keycard_uid): - params = [keycard_uid] - response = self.rpc_request("deleteKeycard", params) - return response - - def delete_keycard_accounts(self, keycard_uid, account_addresses): - params = [keycard_uid, account_addresses] - response = self.rpc_request("deleteKeycardAccounts", params) - return response - - def delete_all_keycards_with_key_uid(self, key_uid): - params = [key_uid] - response = self.rpc_request("deleteAllKeycardsWithKeyUID", params) - return response - - def keycard_locked(self, keycard_uid): - params = [keycard_uid] - response = self.rpc_request("keycardLocked", params) - return response - - def keycard_unlocked(self, keycard_uid): - params = [keycard_uid] - response = self.rpc_request("keycardUnlocked", params) - return response - - def set_keycard_name(self, keycard_uid, kp_name): - params = [keycard_uid, kp_name] - response = self.rpc_request("setKeycardName", params) - return response - - def update_keycard_uid(self, old_keycard_uid, new_keycard_uid): - params = [old_keycard_uid, new_keycard_uid] - response = self.rpc_request("updateKeycardUID", params) - return response - - def migrate_non_profile_keycard_keypair_to_app(self, mnemonic, password): - params = [mnemonic, password] - response = self.rpc_request("migrateNonProfileKeycardKeypairToApp", params) - return response - - def get_random_mnemonic(self): - response = self.rpc_request("getRandomMnemonic", []) - return response - - def get_all_known_keycards(self): - response = self.rpc_request("getAllKnownKeycards", []) - return response - - def get_keycard_by_keycard_uid(self, keycard_uid): - params = [keycard_uid] - response = self.rpc_request("getKeycardByKeycardUID", params) - return response - - def get_keycards_with_same_key_uid(self, key_uid): - params = [key_uid] - response = self.rpc_request("getKeycardsWithSameKeyUID", params) - return response - - def add_keypair_stored_to_keycard(self, key_uid, master_address, name, wallet_accounts): - params = [key_uid, master_address, name, wallet_accounts] - response = self.rpc_request("addKeypairStoredToKeycard", params) - return response - - def update_keypair(self, keypair): - params = [keypair] - response = self.rpc_request("updateKeypair", params) - return response - - def get_watch_only_accounts(self): - response = self.rpc_request("getWatchOnlyAccounts", []) - return response - - def delete_keypair(self, key_uid, password): - params = [key_uid, password] - response = self.rpc_request("deleteKeypair", params) - return response - - def remaining_account_capacity(self): - response = self.rpc_request("remainingAccountCapacity", []) - return response - - def remaining_keypair_capacity(self): - response = self.rpc_request("remainingKeypairCapacity", []) - return response - - def remaining_watch_only_account_capacity(self): - response = self.rpc_request("remainingWatchOnlyAccountCapacity", []) - return response - - def get_num_of_addresses_to_generate_for_keypair(self, key_uid): - params = [key_uid] - response = self.rpc_request("getNumOfAddressesToGenerateForKeypair", params) - return response - - def verify_keystore_file_for_account(self, address, password): - params = [address, password] - response = self.rpc_request("verifyKeystoreFileForAccount", params) - return response diff --git a/clients/services/appgeneral.py b/clients/services/appgeneral.py deleted file mode 100644 index 3a35909..0000000 --- a/clients/services/appgeneral.py +++ /dev/null @@ -1,17 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class AppgeneralService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "appgeneral") - - def get_currencies(self): - params = [] - response = self.rpc_request("getCurrencies", params) - return response - - def version(self): - params = [] - response = self.rpc_request("version", params) - return response diff --git a/clients/services/connector.py b/clients/services/connector.py deleted file mode 100644 index 291c972..0000000 --- a/clients/services/connector.py +++ /dev/null @@ -1,24 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class ConnectorService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "connector") - - def request_accounts_accepted(self, request_id: str, account: str, chain_id: int): - params = { - "requestId": request_id, - "account": account, - "chainId": chain_id, - } - response = self.rpc_request("requestAccountsAccepted", [params]) - return response - - def send_transaction_accepted(self, request_id: str, tx_hash: str): - params = { - "requestId": request_id, - "hash": tx_hash, - } - response = self.rpc_request("sendTransactionAccepted", [params]) - return response diff --git a/clients/services/eth.py b/clients/services/eth.py deleted file mode 100644 index 8860ce7..0000000 --- a/clients/services/eth.py +++ /dev/null @@ -1,12 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class EthService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "eth") - - def estimate_gas(self, id: int, to: str, value: int): - params = params = [id, {"to": to, "value": value}] - response = self.rpc_request("estimateGas", params) - return response diff --git a/clients/services/multiaccounts.py b/clients/services/multiaccounts.py deleted file mode 100644 index a9d21f9..0000000 --- a/clients/services/multiaccounts.py +++ /dev/null @@ -1,17 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class MultiAccountsService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "multiaccounts") - - def store_identity_image(self, key_uid: str, path: str, ax: int, ay: int, bx: int, by: int): - params = [key_uid, path, ax, ay, bx, by] - response = self.rpc_request("storeIdentityImage", params) - return response - - def get_identity_images(self, key_uid: str): - params = [key_uid] - response = self.rpc_request("getIdentityImages", params) - return response diff --git a/clients/services/newsfeed.py b/clients/services/newsfeed.py deleted file mode 100644 index a949c8d..0000000 --- a/clients/services/newsfeed.py +++ /dev/null @@ -1,34 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class NewsFeedService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "newsfeed") - - def enabled(self) -> bool: - """Check if newsfeed is enabled.""" - return self.rpc_request("enabled") - - def set_enabled(self, value: bool) -> None: - """Enable or disable newsfeed.""" - params = [value] - self.rpc_request("setEnabled", params) - - def notifications_enabled(self) -> bool: - """Check if notifications are enabled.""" - return self.rpc_request("notificationsEnabled") - - def set_notifications_enabled(self, value: bool) -> None: - """Enable or disable notifications.""" - params = [value] - self.rpc_request("setNotificationsEnabled", params) - - def rss_enabled(self) -> bool: - """Check if RSS is enabled.""" - return self.rpc_request("rSSEnabled") - - def set_rss_enabled(self, value: bool) -> None: - """Enable or disable RSS.""" - params = [value] - self.rpc_request("setRSSEnabled", params) diff --git a/clients/services/service.py b/clients/services/service.py deleted file mode 100644 index 8583c26..0000000 --- a/clients/services/service.py +++ /dev/null @@ -1,12 +0,0 @@ -from clients.rpc import RpcClient - - -class Service: - def __init__(self, client: RpcClient, name: str): - assert name != "" - self.rpc_client = client - self.name = name - - def rpc_request(self, method: str, params=None): - full_method_name = f"{self.name}_{method}" - return self.rpc_client.rpc_valid_request(full_method_name, params) diff --git a/clients/services/settings.py b/clients/services/settings.py deleted file mode 100644 index d0f99be..0000000 --- a/clients/services/settings.py +++ /dev/null @@ -1,16 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class SettingsService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "settings") - - def get_settings(self): - response = self.rpc_request("getSettings") - return response - - def save_setting(self, key, value): - params = [key, value] - response = self.rpc_request("saveSetting", params) - return response diff --git a/clients/services/sharedurls.py b/clients/services/sharedurls.py deleted file mode 100644 index 615ba93..0000000 --- a/clients/services/sharedurls.py +++ /dev/null @@ -1,47 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class SharedURLsService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "sharedurls") - - def share_community_url_with_chat_key(self, community_id: str): - params = [community_id] - response = self.rpc_request("shareCommunityURLWithChatKey", params) - return response - - def share_community_url_with_data(self, community_id: str): - params = [community_id] - response = self.rpc_request("shareCommunityURLWithData", params) - return response - - def share_community_channel_url_with_chat_key(self, community_id: str, channel_id: str): - params = [community_id, channel_id] - response = self.rpc_request("shareCommunityChannelURLWithChatKey", params) - return response - - def share_community_channel_url_with_data(self, community_id: str, channel_id: str): - params = [community_id, channel_id] - response = self.rpc_request("shareCommunityChannelURLWithData", params) - return response - - def share_user_url_with_ens(self, pub_key: str): - params = [pub_key] - response = self.rpc_request("shareUserURLWithENS", params) - return response - - def share_user_url_with_chat_key(self, pub_key: str): - params = [pub_key] - response = self.rpc_request("shareUserURLWithChatKey", params) - return response - - def share_user_url_with_data(self, pub_key: str): - params = [pub_key] - response = self.rpc_request("shareUserURLWithData", params) - return response - - def parse_shared_url(self, url: str): - params = [url] - response = self.rpc_request("parseSharedURL", params) - return response diff --git a/clients/services/wakuext.py b/clients/services/wakuext.py deleted file mode 100644 index 031b42f..0000000 --- a/clients/services/wakuext.py +++ /dev/null @@ -1,777 +0,0 @@ -from enum import Enum -from typing import TypedDict, Union - -from clients.rpc import RpcClient -from clients.services.service import Service -from resources.enums import MessageContentType -from utils.image_utils import ImageCropRect - - -class PushNotificationRegistrationTokenType(Enum): - UNKNOWN = 0 - APN_TOKEN = 1 - FIREBASE_TOKEN = 2 - - -class ActivityCenterNotificationType(Enum): - NOTIFICATION_NO_TYPE = 0 - NOTIFICATION_TYPE_NEW_ONE_TO_ONE = 1 - NOTIFICATION_TYPE_NEW_PRIVATE_GROUP_CHAT = 2 - NOTIFICATION_TYPE_MENTION = 3 - NOTIFICATION_TYPE_REPLY = 4 - NOTIFICATION_TYPE_CONTACT_REQUEST = 5 - NOTIFICATION_TYPE_COMMUNITY_INVITATION = 6 - NOTIFICATION_TYPE_COMMUNITY_REQUEST = 7 - NOTIFICATION_TYPE_COMMUNITY_MEMBERSHIP_REQUEST = 8 - NOTIFICATION_TYPE_COMMUNITY_KICKED = 9 - NOTIFICATION_TYPE_CONTACT_VERIFICATION = 10 - NOTIFICATION_TYPE_CONTACT_REMOVED = 11 - NOTIFICATION_TYPE_NEW_KEYPAIR_ADDED_TO_PAIRED_DEVICE = 12 - NOTIFICATION_TYPE_OWNER_TOKEN_RECEIVED = 13 - NOTIFICATION_TYPE_OWNERSHIP_RECEIVED = 14 - NOTIFICATION_TYPE_OWNERSHIP_LOST = 15 - NOTIFICATION_TYPE_SET_SIGNER_FAILED = 16 - NOTIFICATION_TYPE_SET_SIGNER_DECLINED = 17 - NOTIFICATION_TYPE_SHARE_ACCOUNTS = 18 - NOTIFICATION_TYPE_COMMUNITY_TOKEN_RECEIVED = 19 - NOTIFICATION_TYPE_FIRST_COMMUNITY_TOKEN_RECEIVED = 20 - NOTIFICATION_TYPE_COMMUNITY_BANNED = 21 - NOTIFICATION_TYPE_COMMUNITY_UNBANNED = 22 - NOTIFICATION_TYPE_NEW_INSTALLATION_RECEIVED = 23 - NOTIFICATION_TYPE_NEW_INSTALLATION_CREATED = 24 - NOTIFICATION_TYPE_BACKUP_SYNCING_FETCHING = 25 - NOTIFICATION_TYPE_BACKUP_SYNCING_SUCCESS = 26 - NOTIFICATION_TYPE_BACKUP_SYNCING_PARTIAL_FAILURE = 27 - NOTIFICATION_TYPE_BACKUP_SYNCING_FAILURE = 28 - NOTIFICATION_TYPE_NEWS = 29 - - -class ActivityCenterMembershipStatus(Enum): - IDLE = 0 - PENDING = 1 - ACCEPTED = 2 - DECLINED = 3 - ACCEPTED_PENDING = 4 - DECLINED_PENDING = 5 - OWNERSHIP_CHANGED = 6 - - -class ActivityCenterQueryParamsRead(Enum): - READ = 1 - UNREAD = 2 - ALL = 3 - - -class ContactRequestState(Enum): - NONE = 0 - MUTUAL = 1 - SENT = 2 - RECEIVED = 3 - DISMISSED = 4 - - -class SendPinMessagePayload(TypedDict): - chat_id: str - message_id: str - pinned: bool - - -class SendChatMessagePayload(TypedDict): - chat_id: str - text: str - content_type: int - - -class CommunityPermissionsAccess(Enum): - UNKNOWN = 0 - AUTO_ACCEPT = 1 - MANUAL_ACCEPT = 3 - - -class Error(Exception): - def __init__(self, message): - self.message = message - - -class WakuextService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "wakuext") - - def start_messenger(self): - self.rpc_request("startMessenger") - - def send_contact_request(self, contact_id: str, message: str): - params = [{"id": contact_id, "message": message}] - response = self.rpc_request("sendContactRequest", params) - return response - - def accept_contact_request(self, request_id: str): - params = [{"id": request_id}] - response = self.rpc_request("acceptContactRequest", params) - return response - - def accept_latest_contact_request_for_contact(self, request_id: str): - params = [{"id": request_id}] - response = self.rpc_request("acceptLatestContactRequestForContact", params) - return response - - def decline_contact_request(self, request_id: str): - params = [{"id": request_id}] - response = self.rpc_request("declineContactRequest", params) - return response - - def dismiss_latest_contact_request_for_contact(self, request_id: str): - params = [{"id": request_id}] - response = self.rpc_request("dismissLatestContactRequestForContact", params) - return response - - def get_latest_contact_request_for_contact(self, request_id: str): - params = [request_id] - response = self.rpc_request("getLatestContactRequestForContact", params) - return response - - def retract_contact_request(self, request_id: str): - params = [{"id": request_id}] - response = self.rpc_request("retractContactRequest", params) - return response - - def remove_contact(self, request_id: str): - params = [request_id] - response = self.rpc_request("removeContact", params) - return response - - def set_contact_local_nickname(self, request_id: str, nickname: str): - params = [{"id": request_id, "nickname": nickname}] - response = self.rpc_request("setContactLocalNickname", params) - return response - - def get_contacts(self): - response = self.rpc_request("contacts") - return response - - def get_contact_by_id(self, id: str): - params = [id] - response = self.rpc_request("getContactByID", params) - return response - - def add_contact(self, contact_id: str, displayName: str): - params = [{"id": contact_id, "nickname": "fake_nickname", "displayName": displayName, "ensName": ""}] - response = self.rpc_request("addContact", params) - return response - - def send_one_to_one_message(self, contact_id: str, message: str): - params = [{"id": contact_id, "message": message}] - response = self.rpc_request("sendOneToOneMessage", params) - return response - - def create_group_chat_with_members(self, pubkey_list: list, group_chat_name: str): - params = [group_chat_name, pubkey_list] - response = self.rpc_request("createGroupChatWithMembers", params) - return response - - def send_group_chat_message(self, group_id: str, message: str): - params = [{"id": group_id, "message": message}] - response = self.rpc_request("sendGroupChatMessage", params) - return response - - def leave_group_chat(self, chat_id: str, remove: bool): - params = [chat_id, remove] - response = self.rpc_request("leaveGroupChat", params) - return response - - def create_group_chat_from_invitation(self, name: str, chat_id: str, admin_pk: str): - params = [name, chat_id, admin_pk] - response = self.rpc_request("createGroupChatFromInvitation", params) - return response - - def add_members_to_group_chat(self, chat_id: str, members: list): - params = [chat_id, members] - response = self.rpc_request("addMembersToGroupChat", params) - return response - - def remove_member_from_group_chat(self, chat_id: str, member: str): - params = [chat_id, member] - response = self.rpc_request("removeMemberFromGroupChat", params) - return response - - def remove_members_from_group_chat(self, chat_id: str, members: list): - params = [chat_id, members] - response = self.rpc_request("removeMembersFromGroupChat", params) - return response - - def confirm_joining_group(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("confirmJoiningGroup", params) - return response - - def change_group_chat_name(self, chat_id: str, name: str): - params = [chat_id, name] - response = self.rpc_request("changeGroupChatName", params) - return response - - def send_group_chat_invitation_request(self, chat_id: str, admin_pk: str, message: str): - params = [chat_id, admin_pk, message] - response = self.rpc_request("sendGroupChatInvitationRequest", params) - return response - - def get_group_chat_invitations(self): - response = self.rpc_request("getGroupChatInvitations") - return response - - def send_group_chat_invitation_rejection(self, invitation_request_id: str): - params = [invitation_request_id] - response = self.rpc_request("sendGroupChatInvitationRejection", params) - return response - - def create_community( - self, - name, - description, - color="#ffffff", - membership: CommunityPermissionsAccess = CommunityPermissionsAccess.AUTO_ACCEPT, - image="", - image_rect=ImageCropRect(), - ): - params = { - "membership": membership.value, - "name": name, - "color": color, - "description": description, - "image": image, - "imageAx": image_rect.ax, - "imageAy": image_rect.ay, - "imageBx": image_rect.bx, - "imageBy": image_rect.by, - } - response = self.rpc_request("createCommunity", [params]) - return response - - def edit_community( - self, - community_id, - name, - color="#ffffff", - membership: CommunityPermissionsAccess = CommunityPermissionsAccess.AUTO_ACCEPT, - description="", - image="", - image_rect=ImageCropRect(), - ): - params = { - "CommunityID": community_id, - "membership": membership.value, - "name": name, - "color": color, - "description": description, - "image": image, - "imageAx": image_rect.ax, - "imageAy": image_rect.ay, - "imageBx": image_rect.bx, - "imageBy": image_rect.by, - } - response = self.rpc_request("editCommunity", [params]) - return response - - def fetch_community(self, community_key): - params = [{"communityKey": community_key, "waitForResponse": True, "tryDatabase": True}] - response = self.rpc_request("fetchCommunity", params) - return response - - def request_to_join_community(self, community_id, address="fakeaddress"): - params = [{"communityId": community_id, "addressesToReveal": [address], "airdropAddress": address}] - response = self.rpc_request("requestToJoinCommunity", params) - return response - - def accept_request_to_join_community(self, request_to_join_id: str): - params = [{"id": request_to_join_id}] - response = self.rpc_request("acceptRequestToJoinCommunity", params) - return response - - def cancel_request_to_join_community(self, request_to_join_id: str): - params = [{"id": request_to_join_id}] - response = self.rpc_request("cancelRequestToJoinCommunity", params) - return response - - def decline_request_to_join_community(self, request_to_join_id: str): - params = [{"id": request_to_join_id}] - response = self.rpc_request("declineRequestToJoinCommunity", params) - return response - - def canceled_requests_to_join_for_community(self, community_id: str): - params = [community_id] - response = self.rpc_request("canceledRequestsToJoinForCommunity", params) - return response - - def pending_requests_to_join_for_community(self, community_id: str): - params = [community_id] - response = self.rpc_request("pendingRequestsToJoinForCommunity", params) - return response - - def declined_requests_to_join_for_community(self, community_id: str): - params = [community_id] - response = self.rpc_request("declinedRequestsToJoinForCommunity", params) - return response - - def latest_request_to_join_for_community(self, community_id: str): - params = [community_id] - response = self.rpc_request("latestRequestToJoinForCommunity", params) - return response - - def my_pending_requests_to_join(self): - params = [] - response = self.rpc_request("myPendingRequestsToJoin", params) - return response - - def my_canceled_requests_to_join(self): - params = [] - response = self.rpc_request("myCanceledRequestsToJoin", params) - return response - - def check_and_delete_pending_request_to_join_community(self): - params = [] - response = self.rpc_request("checkAndDeletePendingRequestToJoinCommunity", params) - return response - - def all_non_approved_communities_requests_to_join(self): - params = [] - response = self.rpc_request("allNonApprovedCommunitiesRequestsToJoin", params) - return response - - def check_permissions_to_join_community(self, community_id: str): - params = [{"communityId": community_id}] - response = self.rpc_request("checkPermissionsToJoinCommunity", params) - return response - - def generate_joining_community_requests_for_signing(self, member_pub_key: str, community_id: str, addresses_to_reveal: list): - params = [member_pub_key, community_id, addresses_to_reveal] - response = self.rpc_request("generateJoiningCommunityRequestsForSigning", params) - return response - - def generate_edit_community_requests_for_signing(self, member_pub_key: str, community_id: str, addresses_to_reveal: list): - params = [member_pub_key, community_id, addresses_to_reveal] - response = self.rpc_request("generateEditCommunityRequestsForSigning", params) - return response - - def send_chat_message(self, chat_id, message, content_type=MessageContentType.TEXT_PLAIN.value, responseTo: str = ""): - params = [ - { - "chatId": chat_id, - "text": message, - "contentType": content_type, - "responseTo": responseTo, - } - ] - response = self.rpc_request("sendChatMessage", params) - return response - - def send_chat_messages(self, messages: list[SendChatMessagePayload]): - params = [[{"chatId": m["chat_id"], "text": m["text"], "contentType": m["content_type"]} for m in messages]] - response = self.rpc_request("sendChatMessages", params) - return response - - def resend_chat_message(self, message_id: str): - params = [message_id] - response = self.rpc_request("reSendChatMessage", params) - return response - - def leave_community(self, community_id): - params = [community_id] - response = self.rpc_request("leaveCommunity", params) - return response - - def set_light_client(self, enabled=True): - params = [{"enabled": enabled}] - response = self.rpc_request("setLightClient", params) - return response - - def peers(self): - params = [] - response = self.rpc_request("peers", params) - return response - - def chat_messages(self, chat_id: str, cursor="", limit=10): - params = [chat_id, cursor, limit] - response = self.rpc_request("chatMessages", params) - return response - - def message_by_message_id(self, message_id: str): - params = [message_id] - response = self.rpc_request("messageByMessageID", params) - return response - - def all_messages_from_chat_which_match_term(self, chat_id: str, searchTerm: str, caseSensitive: bool): - params = [chat_id, searchTerm, caseSensitive] - response = self.rpc_request("allMessagesFromChatWhichMatchTerm", params) - return response - - def all_messages_from_chats_and_communities_which_match_term( - self, community_ids: list[str], chat_ids: list[str], searchTerm: str, caseSensitive: bool - ): - params = [community_ids, chat_ids, searchTerm, caseSensitive] - response = self.rpc_request("allMessagesFromChatsAndCommunitiesWhichMatchTerm", params) - return response - - def send_pin_message(self, message: SendPinMessagePayload): - params = [message] - response = self.rpc_request("sendPinMessage", params) - return response - - def chat_pinned_messages(self, chat_id: str, cursor="", limit=10): - params = [chat_id, cursor, limit] - response = self.rpc_request("chatPinnedMessages", params) - return response - - def set_user_status(self, new_status: int, custom_text=""): - params = [new_status, custom_text] - response = self.rpc_request("setUserStatus", params) - return response - - def set_bio(self, bio: str): - params = [bio] - response = self.rpc_request("setBio", params) - return response - - def set_customization_color(self, color: str, key_uid: str): - params = [{"customizationColor": color, "keyUid": key_uid}] - response = self.rpc_request("setCustomizationColor", params) - return response - - def set_syncing_on_mobile_network(self, enabled: bool): - params = [{"enabled": enabled}] - response = self.rpc_request("setSyncingOnMobileNetwork", params) - return response - - def status_updates(self): - params = [] - response = self.rpc_request("statusUpdates", params) - return response - - def edit_message(self, message_id: str, new_text: str): - params = [{"id": message_id, "text": new_text}] - response = self.rpc_request("editMessage", params) - return response - - def delete_message(self, message_id: str): - params = [message_id] - response = self.rpc_request("deleteMessage", params) - return response - - def delete_messages_by_chat_id(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("deleteMessagesByChatID", params) - return response - - def delete_message_and_send(self, message_id: str): - params = [message_id] - response = self.rpc_request("deleteMessageAndSend", params) - return response - - def delete_message_for_me_and_sync(self, local_chat_id: str, message_id: str): - params = [local_chat_id, message_id] - response = self.rpc_request("deleteMessageForMeAndSync", params) - return response - - def mark_message_as_unread(self, chat_id: str, message_id: str): - params = [chat_id, message_id] - response = self.rpc_request("markMessageAsUnread", params) - return response - - def first_unseen_message_id(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("firstUnseenMessageID", params) - return response - - def update_message_outgoing_status(self, message_id: str, new_status: str): - params = [message_id, new_status] - response = self.rpc_request("updateMessageOutgoingStatus", params) - return response - - def chats(self): - params = [] - response = self.rpc_request("chats", params) - return response - - def chat(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("chat", params) - return response - - def chats_preview(self, filter_type: int): - params = [filter_type] - response = self.rpc_request("chatsPreview", params) - return response - - def active_chats(self): - params = [] - response = self.rpc_request("activeChats", params) - return response - - def mute_chat(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("muteChat", params) - return response - - def mute_chat_v2(self, chat_id: str, muted_type: int): - params = [{"ChatId": chat_id, "MutedType": muted_type}] - response = self.rpc_request("muteChatV2", params) - return response - - def unmute_chat(self, chat_id: str): - params = [chat_id] - response = self.rpc_request("unmuteChat", params) - return response - - def clear_history(self, chat_id: str): - params = [{"id": chat_id}] - response = self.rpc_request("clearHistory", params) - return response - - def deactivate_chat(self, chat_id: str, preserve_history: bool): - params = [{"id": chat_id, "preserveHistory": preserve_history}] - response = self.rpc_request("deactivateChat", params) - return response - - def save_chat(self, chat_id: str, active=True): - params = [{"id": chat_id, "active": active}] - response = self.rpc_request("saveChat", params) - return response - - def create_one_to_one_chat(self, chat_id: str, ens_name: str): - params = [{"id": chat_id, "ensName": ens_name}] - response = self.rpc_request("createOneToOneChat", params) - return response - - def register_for_push_notifications(self, device_token: str, apnTopic: str, tokenType: PushNotificationRegistrationTokenType): - params = [device_token, apnTopic, tokenType.value] - response = self.rpc_request("registerForPushNotifications", params) - return response - - def get_activity_center_notifications( - self, - activity_types: list = list(ActivityCenterNotificationType), - read_type: Union[ActivityCenterQueryParamsRead, None] = None, - cursor: str = "", - limit: int = 20, - ): - params = { - "activityTypes": [item.value for item in activity_types], - "cursor": cursor, - "limit": limit, - } - if read_type is not None: - params["readType"] = read_type - - response = self.rpc_request(method="activityCenterNotifications", params=[params]) - return response - - def activity_center_notifications_count( - self, activity_types: list = list(ActivityCenterNotificationType), read_type: Union[ActivityCenterQueryParamsRead, None] = None - ): - params = [{"activityTypes": activity_types, "readType": read_type}] - response = self.rpc_request("activityCenterNotificationsCount", params) - return response - - def has_unseen_activity_center_notifications(self): - params = [] - response = self.rpc_request("hasUnseenActivityCenterNotifications", params) - return response - - def mark_as_seen_activity_center_notifications(self): - params = [] - response = self.rpc_request("markAsSeenActivityCenterNotifications", params) - return response - - def mark_activity_center_notifications_read(self, message_id: str): - params = [message_id] - response = self.rpc_request("markActivityCenterNotificationsRead", params) - return response - - def mark_activity_center_notifications_unread(self, message_id: str): - params = [message_id] - response = self.rpc_request("markActivityCenterNotificationsUnread", params) - return response - - def mark_all_activity_center_notifications_read(self): - params = [] - response = self.rpc_request("markAllActivityCenterNotificationsRead", params) - return response - - def accept_activity_center_notifications(self, message_id: str): - params = [message_id] - response = self.rpc_request("acceptActivityCenterNotifications", params) - return response - - def dismiss_activity_center_notifications(self, message_id: str): - params = [message_id] - response = self.rpc_request("dismissActivityCenterNotifications", params) - return response - - def delete_activity_center_notifications(self, message_id: str): - params = [message_id] - response = self.rpc_request("deleteActivityCenterNotifications", params) - return response - - def get_activity_center_state(self): - params = [] - response = self.rpc_request("getActivityCenterState", params) - return response - - def peer_id(self): - params = [] - response = self.rpc_request("peerID", params) - return response - - def send_emoji_reaction(self, receiver_chat_id: str, message_id: str, emoji_id: int): - params = [receiver_chat_id, message_id, emoji_id] - response = self.rpc_request(method="sendEmojiReaction", params=params) - return response - - def send_emoji_reaction_v2(self, chat_id: str, message_id: str, emoji: str): - params = [chat_id, message_id, emoji] - response = self.rpc_request(method="sendEmojiReactionV2", params=params) - return response - - def send_emoji_reaction_retraction(self, last_emoji_id: str): - params = [last_emoji_id] - response = self.rpc_request(method="sendEmojiReactionRetraction", params=params) - return response - - def emoji_reactions_by_chat_id(self, sender_chat_id: str, limit: int): - params = [sender_chat_id, None, limit] - response = self.rpc_request(method="emojiReactionsByChatID", params=params) - return response - - def emoji_reactions_by_chat_id_message_id(self, sender_chat_id: str, message_id: str): - params = [sender_chat_id, message_id] - response = self.rpc_request(method="emojiReactionsByChatIDMessageID", params=params) - return response - - def get_saved_addresses(self, params=[]): - response = self.rpc_request("getSavedAddresses", params) - return response - - def get_saved_addresses_per_mode(self, is_test: bool): - params = [is_test] - response = self.rpc_request("getSavedAddressesPerMode", params) - return response - - def upsert_saved_address(self, address: str, name: str, color_id: str, ens: str = "", chain_short_names: str = "", is_test: bool = False): - params = [ - { - "address": address, - "name": name, - "ens": ens, - "colorId": color_id, - "isTest": is_test, - "chainShortNames": chain_short_names, - }, - ] - response = self.rpc_request("upsertSavedAddress", params) - return response - - def delete_saved_address(self, address: str, is_test: bool): - params = [address, is_test] - response = self.rpc_request("deleteSavedAddress", params) - return response - - def remaining_capacity_for_saved_addresses(self, is_test: bool): - params = [is_test] - response = self.rpc_request("remainingCapacityForSavedAddresses", params) - return response - - def set_display_name(self, name: str): - params = [name] - response = self.rpc_request("setDisplayName", params) - return response - - def set_profile_showcase_preferences(self, prefs: dict): - response = self.rpc_request("setProfileShowcasePreferences", [prefs]) - return response - - def get_profile_showcase_preferences(self): - params = [] - response = self.rpc_request("getProfileShowcasePreferences", params) - return response - - def create_community_from_payload(self, community: dict): - params = [community] - response = self.rpc_request("createCommunity", params) - return response - - def communities(self): - params = [] - response = self.rpc_request("communities", params) - return response - - def joined_communities(self): - params = [] - response = self.rpc_request("joinedCommunities", params) - return response - - def log_test(self): - response = self.rpc_request("logTest") - return response - - def create_community_chat(self, community_id: str, c: dict): - params = [community_id, c] - response = self.rpc_request("createCommunityChat", params) - return response - - def edit_community_chat(self, community_id: str, chat_id: str, c: dict): - params = [community_id, chat_id, c] - response = self.rpc_request("editCommunityChat", params) - return response - - def delete_community_chat(self, community_id: str, chat_id: str): - params = [community_id, chat_id] - response = self.rpc_request("deleteCommunityChat", params) - return response - - def reorder_community_chat(self, community_id: str, chat_id: str, position: int): - params = [{"communityId": community_id, "chatId": chat_id, "position": position}] - response = self.rpc_request("reorderCommunityChat", params) - return response - - def mute_community_chats(self, community_id: str, muted_type): - params = [{"communityId": community_id, "mutedType": muted_type}] - response = self.rpc_request("muteCommunityChats", params) - return response - - def un_mute_community_chats(self, community_id: str): - params = [community_id] - response = self.rpc_request("unMuteCommunityChats", params) - return response - - def accept_contact_verification_request(self, id: str, response: str): - params = [id, response] - response = self.rpc_request("acceptContactVerificationRequest", params) - return response - - def decline_contact_verification_request(self, id: str): - params = [id] - response = self.rpc_request("declineContactVerificationRequest", params) - return response - - def cancel_verification_request(self, id: str): - params = [id] - response = self.rpc_request("cancelVerificationRequest", params) - return response - - def get_latest_verification_request_from(self, contact_id: str): - params = [contact_id] - response = self.rpc_request("getLatestVerificationRequestFrom", params) - return response - - def send_contact_verification_request(self, contact_id: str, challenge: str): - params = [contact_id, challenge] - response = self.rpc_request("sendContactVerificationRequest", params) - return response - - def get_received_verification_requests(self): - params = [] - response = self.rpc_request("getReceivedVerificationRequests", params) - return response - - def get_verification_request_sent_to(self, contact_id: str): - params = [contact_id] - response = self.rpc_request("getVerificationRequestSentTo", params) - return response diff --git a/clients/services/wallet.py b/clients/services/wallet.py deleted file mode 100644 index ffd6760..0000000 --- a/clients/services/wallet.py +++ /dev/null @@ -1,94 +0,0 @@ -from clients.rpc import RpcClient -from clients.services.service import Service - - -class WalletService(Service): - def __init__(self, client: RpcClient): - super().__init__(client, "wallet") - - def get_balances_at_by_chain(self, chains: list, addresses: list, tokens: list): - params = [chains, addresses, tokens] - return self.rpc_request("getBalancesByChain", params) - - def start_wallet(self): - return self.rpc_request("startWallet") - - def get_derived_addresses_for_mnemonic(self, mnemonic: str, paths: list): - params = [mnemonic, paths] - return self.rpc_request("getDerivedAddressesForMnemonic", params) - - def send_router_transactions_with_signatures(self, uuid: str, tx_signatures: dict): - params = [{"uuid": uuid, "Signatures": tx_signatures}] - return self.rpc_request("sendRouterTransactionsWithSignatures", params) - - def get_owned_collectibles_async(self, params: dict): - return self.rpc_request("getOwnedCollectiblesAsync", params) - - def start_activity_filter_session_v2(self, params: dict): - return self.rpc_request("startActivityFilterSessionV2", params) - - def reset_activity_filter_session(self, session_id: int): - params = [session_id] - return self.rpc_request("resetActivityFilterSession", params) - - def set_fee_mode(self, path_tx_identity: dict, gas_fee_mode: int): - params = [path_tx_identity, gas_fee_mode] - return self.rpc_request("setFeeMode", params) - - def set_custom_tx_details(self, tx_identity_params: dict, tx_custom_params: dict): - params = [tx_identity_params, tx_custom_params] - return self.rpc_request("setCustomTxDetails", params) - - def get_suggested_routes_async(self, params: dict): - return self.rpc_request("getSuggestedRoutesAsync", params) - - def build_transactions_from_route(self, uuid: str): - params = [uuid] - return self.rpc_request("buildTransactionsFromRoute", params) - - def sign_message(self, hash: str, address: str, password: str): - params = [hash, address, password] - return self.rpc_request("signMessage", params) - - def get_ethereum_chain( - self, - ): - return self.rpc_request("getEthereumChains") - - def get_token_list( - self, - ): - return self.rpc_request("getTokenList") - - def get_crypto_on_ramps( - self, - ): - return self.rpc_request("getCryptoOnRamps") - - def get_cached_currency_formats( - self, - ): - return self.rpc_request("getCachedCurrencyFormats") - - def fetch_prices(self, symbols: list, currencies: list): - params = [symbols, currencies] - return self.rpc_request("fetchPrices", params) - - def fetch_market_values(self, symbols: list, currency: str): - params = [symbols, currency] - return self.rpc_request("fetchMarketValues", params) - - def fetch_token_details(self, symbols: list): - params = [symbols] - return self.rpc_request("fetchTokenDetails", params) - - def get_wallet_connect_active_sessions(self, timestamp: int): - params = [timestamp] - return self.rpc_request("getWalletConnectActiveSessions", params) - - def stop_suggested_routes_async_calculation(self): - return self.rpc_request("stopSuggestedRoutesAsyncCalculation") - - def fetch_or_get_cached_wallet_balances(self, addresses: list, force_refresh: bool = False): - params = [addresses, force_refresh] - return self.rpc_request("fetchOrGetCachedWalletBalances", params) diff --git a/clients/signals.py b/clients/signals.py deleted file mode 100644 index 10339d4..0000000 --- a/clients/signals.py +++ /dev/null @@ -1,226 +0,0 @@ -import json -import logging -import os -import threading -import time -from datetime import datetime -from enum import Enum -from pathlib import Path - -import websocket - -from resources.constants import SIGNALS_DIR, LOG_SIGNALS_TO_FILE - - -# Only signals defined in SignalType are processed by SignalClient -class SignalType(Enum): - MESSAGES_NEW = "messages.new" - MESSAGE_DELIVERED = "message.delivered" - NODE_READY = "node.ready" - NODE_STARTED = "node.started" - NODE_LOGIN = "node.login" - NODE_STOPPED = "node.stopped" - MEDIASERVER_STARTED = "mediaserver.started" - WALLET = "wallet" - WALLET_SUGGESTED_ROUTES = "wallet.suggested.routes" - WALLET_ROUTER_SIGN_TRANSACTIONS = "wallet.router.sign-transactions" - WALLET_ROUTER_SENDING_TRANSACTIONS_STARTED = "wallet.router.sending-transactions-started" - WALLET_ROUTER_TRANSACTIONS_SENT = "wallet.router.transactions-sent" - LOCAL_PAIRING = "localPairing" - DB_REENCRYPTION_STARTED = "db.reEncryption.started" - DB_REENCRYPTION_FINISHED = "db.reEncryption.finished" - CONNECTOR_SEND_REQUEST_ACCOUNTS = "connector.sendRequestAccounts" - CONNECTOR_SEND_TRANSACTION = "connector.sendTransaction" - CONNECTOR_SIGN = "connector.sign" - CONNECTOR_DAPP_PERMISSION_GRANTED = "connector.dAppPermissionGranted" - CONNECTOR_DAPP_PERMISSION_REVOKED = "connector.dAppPermissionRevoked" - CONNECTOR_DAPP_CHAIN_ID_SWITCHED = "connector.dAppChainIdSwitched" - - -class WalletEventType(Enum): - WALLET_ACTIVITY_FILTERING_DONE = "wallet-activity-filtering-done" - WALLET_ACTIVITY_FILTERING_ENTRIES_UPDATED = "wallet-activity-filtering-entries-updated" - WALLET_ACTIVITY_SESSION_UPDATED = "wallet-activity-session-updated" - TRANSACTIONS_PENDING_TRANSACTION_UPDATE = "pending-transaction-update" - TRANSACTIONS_PENDING_TRANSACTION_STATUS_CHANGED = "pending-transaction-status-changed" - WALLET_TICK_RELOAD = "wallet-tick-reload" - - -class LocalPairingEventType(Enum): - # Both Sender and Receiver - EVENT_PEER_DISCOVERED = "peer-discovered" - EVENT_CONNECTION_ERROR = "connection-error" - EVENT_CONNECTION_SUCCESS = "connection-success" - EVENT_TRANSFER_ERROR = "transfer-error" - EVENT_TRANSFER_SUCCESS = "transfer-success" - EVENT_RECEIVED_INSTALLATION = "received-installation" - # Only Receiver side - EVENT_RECEIVED_ACCOUNT = "received-account" - EVENT_PROCESS_SUCCESS = "process-success" - EVENT_PROCESS_ERROR = "process-error" - EVENT_RECEIVED_KEYSTORE_FILES = "received-keystore-files" - - -class LocalPairingEventAction(Enum): - ACTION_CONNECT = 1 - ACTION_PAIRING_ACCOUNT = 2 - ACTION_SYNC_DEVICE = 3 - ACTION_PAIRING_INSTALLATION = 4 - ACTION_PEER_DISCOVERY = 5 - ACTION_KEYSTORE_FILES_TRANSFER = 6 - - -class SignalClient: - def __init__(self, ws_url): - self.url = f"{ws_url}/signals" - - self.received_signals = { - # For each signal type, store: - # - list of received signals - # - expected received event delta count (resets to 1 after each wait_for_event call) - # - expected received event count - # - a function that takes the received signal as an argument and returns True if the signal is accepted (counted) or discarded - signal: { - "received": [], - "delta_count": 1, - "expected_count": 1, - "accept_fn": None, - } - for signal in SignalType - } - if LOG_SIGNALS_TO_FILE: - self.signal_file_path = os.path.join( - SIGNALS_DIR, - f"signal_{ws_url.split(':')[-1]}_{datetime.now().strftime('%H%M%S')}.log", - ) - Path(SIGNALS_DIR).mkdir(parents=True, exist_ok=True) - - def on_message(self, ws, signal): - signal_data = json.loads(signal) - if LOG_SIGNALS_TO_FILE: - self.write_signal_to_file(signal_data) - - signal_type = signal_data.get("type") - try: - signal_type = self._convert_signal_type(signal_type) - except ValueError: - # Ignore unregistered signal types - return - - if signal_type not in self.received_signals: - # This should never happen, as we register all signal types from SignalType enum - raise ValueError(f"Signal type {signal_type} is not registered") - - accept_fn = self.received_signals[signal_type]["accept_fn"] - if not accept_fn or accept_fn(signal_data): - self.received_signals[signal_type]["received"].append(signal_data) - - # TODO: This is a temporary workaround until all tests are migrated to use SignalType enum - @staticmethod - def _convert_signal_type(signal_type: SignalType | str) -> SignalType: - if isinstance(signal_type, SignalType): - return signal_type - if isinstance(signal_type, str): - return SignalType(signal_type) - - # Used to set up how many instances of a signal to wait for, before triggering the actions - # that cause them to be emitted. - def prepare_wait_for_signal(self, signal_type: SignalType, delta_count: int, accept_fn=None): - signal_type = self._convert_signal_type(signal_type) - - if delta_count < 1: - raise ValueError("delta_count must be greater than 0") - self.received_signals[signal_type]["delta_count"] = delta_count - self.received_signals[signal_type]["expected_count"] = len(self.received_signals[signal_type]["received"]) + delta_count - self.received_signals[signal_type]["accept_fn"] = accept_fn - - def wait_for_signal(self, signal_type: SignalType | str, timeout: int | None = 20): - signal_type = self._convert_signal_type(signal_type) - - start_time = time.time() - received_signals = self.received_signals.get(signal_type) - while (not received_signals) or len(received_signals["received"]) < received_signals["expected_count"]: - if timeout is not None and time.time() - start_time >= timeout: - raise TimeoutError(f"Signal {signal_type} is not received in {timeout} seconds") - time.sleep(0.2) - logging.debug(f"Signal {signal_type} is received in {round(time.time() - start_time)} seconds") - delta_count = received_signals["delta_count"] - self.prepare_wait_for_signal(signal_type, 1) - if delta_count == 1: - return self.received_signals[signal_type]["received"][-1] - return self.received_signals[signal_type]["received"][-delta_count:] - - def wait_for_signal_predicate(self, signal_type: SignalType | str, predicate=lambda signal: True, timeout=20): - signal_type = self._convert_signal_type(signal_type) - start_time = time.time() - while True: - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - break - remaining_time = int(timeout - elapsed_time) - signal = self.wait_for_signal(signal_type, remaining_time) - try: - if predicate(signal): - return signal - except Exception as ex: - logging.warning(f"Could not filter signal by predicate because of error: {str(ex)}") - continue - raise TimeoutError(f"Signal {signal_type} satisfying the predicate is not received in {timeout} seconds") - - def wait_for_logout(self): - signal = self.wait_for_signal(SignalType.NODE_STOPPED) - return signal - - def find_signal_containing_pattern(self, signal_type: SignalType | str, event_pattern, timeout=20): - signal_type = self._convert_signal_type(signal_type) - - start_time = time.time() - while True: - if time.time() - start_time >= timeout: - raise TimeoutError(f"Signal {signal_type} containing {event_pattern} is not received in {timeout} seconds") - if not self.received_signals.get(signal_type): - time.sleep(0.2) - continue - for event in self.received_signals[signal_type]["received"]: - if event_pattern in json.dumps(event): - logging.debug(f"Signal {signal_type} containing {event_pattern} is received in {round(time.time() - start_time)} seconds") - return event - time.sleep(0.2) - - def get_all_events(self, signal_type: SignalType | str): - signal_type = self._convert_signal_type(signal_type) - signals = self.received_signals.get(signal_type, {}).get("received", []) - return [signal.get("event") for signal in signals] - - def _on_error(self, ws, error): - logging.error(f"SignalClient [{self.url}]: websocket error: {error}") - - def _on_close(self, ws, close_status_code, close_msg): - logging.debug(f"SignalClient [{self.url}]: websocket connection closed: {close_status_code}, {close_msg}") - - def _on_open(self, ws): - logging.debug(f"SignalClient [{self.url}]: websocket connection opened") - - def _connect(self): - self.wsapp = websocket.WebSocketApp( - url=self.url, - on_message=self.on_message, - on_error=self._on_error, - on_open=self._on_open, - on_close=self._on_close, - ) - self.wsapp.run_forever() - - def connect(self): - websocket_thread = threading.Thread(target=self._connect) - websocket_thread.daemon = True - websocket_thread.start() - - def disconnect(self): - if hasattr(self, "wsapp") and self.wsapp is not None: - self.wsapp.close() - - def write_signal_to_file(self, signal_data): - with open(self.signal_file_path, "a+") as file: - json.dump(signal_data, file) - file.write("\n") diff --git a/clients/status_backend.py b/clients/status_backend.py deleted file mode 100644 index 4b246c5..0000000 --- a/clients/status_backend.py +++ /dev/null @@ -1,524 +0,0 @@ -import json -import logging -import os -import tempfile -import time -import uuid - -import qrcode -import requests -from tenacity import retry, stop_after_delay, wait_fixed, wait_exponential, retry_if_exception_type - -import resources.constants as constants -from clients.api import ApiClient -from clients.expvar import ExpvarClient -from clients.metrics import Events, StatusGoMetrics -from clients.rpc import RpcClient -from clients.services.accounts import AccountService -from clients.services.appgeneral import AppgeneralService -from clients.services.connector import ConnectorService -from clients.services.eth import EthService -from clients.services.multiaccounts import MultiAccountsService -from clients.services.newsfeed import NewsFeedService -from clients.services.settings import SettingsService -from clients.services.sharedurls import SharedURLsService -from clients.services.wakuext import ( - WakuextService, - PushNotificationRegistrationTokenType, -) -from clients.services.wallet import WalletService -from clients.signals import SignalClient, SignalType -from clients.statusgo_container import StatusBackendContainer -from resources.constants import USE_IPV6, user_1, ANVIL_NETWORK_ID -from utils import fake -from utils import keys -from utils.config import Config - -NANOSECONDS_PER_SECOND = 1_000_000_000 - - -class StatusBackend(RpcClient, SignalClient, ApiClient): - container = None - - def __init__(self, privileged=False, ipv6=USE_IPV6, **kwargs): - self.temp_dir = None - self.ipv6 = True if ipv6 == "Yes" else False - logging.debug(f"Flag USE_IPV6 is: {self.ipv6}") - - url = None - if kwargs.__contains__("url"): - url = kwargs.get("url", "") - elif Config.status_backend_urls: - try: - url = next(Config.status_backend_urls) - except StopIteration: - raise Exception("--status-backend-url is found, but not enough backends provided") - - data_dir = kwargs.get("data_dir", None) # TODO: Should be argument of `init_status_backend` or fetched from the app - self.logLevel = kwargs.get("logLevel", "DEBUG") - - if url: - assert url != "", "not enough status-backend urls provided" - if data_dir is None: - self.temp_dir = tempfile.TemporaryDirectory() - self.data_dir = self.temp_dir.name - else: - self.data_dir = data_dir - if kwargs.get("connector_enabled", False): - self.connector_ws_url = f"ws://localhost:{constants.STATUS_CONNECTOR_WS_PORT}" - else: - self.container = StatusBackendContainer(privileged, self.ipv6, **kwargs) - self.temp_dir = None - self.data_dir = self.container.data_dir() - url = self.container.url - if kwargs.get("connector_enabled", False): - self.connector_ws_url = self.container.connector_ws_url - - assert self.data_dir != "" - self.base_url = url - self.api_url = f"{url}/statusgo" - self.ws_url = f"{url}".replace("http", "ws") - self.public_key = "" - self.mnemonic = "" - self.key_uid = "" - self.password = "" - self.display_name = "" - self.device_id = str(uuid.uuid4()) # In reality this is taken from the device, don't confuse with Status installation_id - self.device_platform = PushNotificationRegistrationTokenType.UNKNOWN - self.node_login_event = {} - self.events = Events() - self.version = "unknown" - self.network_id = 1 - - RpcClient.__init__(self) - ApiClient.__init__(self, self.api_url) - SignalClient.__init__(self, self.ws_url) - - self.wait_for_healthy() - - SignalClient.connect(self) - - self.wallet_service = WalletService(self) - self.wakuext_service = WakuextService(self) - self.accounts_service = AccountService(self) - self.newsfeed_service = NewsFeedService(self) - self.multiaccounts_service = MultiAccountsService(self) - self.settings_service = SettingsService(self) - self.sharedurls_service = SharedURLsService(self) - self.connector_service = ConnectorService(self) - self.appgeneral_service = AppgeneralService(self) - self.eth_service = EthService(self) - self.expvar_client = ExpvarClient(self.base_url) - - def __del__(self): - self.shutdown() - - def shutdown(self, log_sufix=""): - SignalClient.disconnect(self) - - if self.container: - self.container.shutdown(log_sufix) - - if self.temp_dir is not None: - self.temp_dir.cleanup() - - @retry( - stop=stop_after_delay(10), - wait=wait_exponential(multiplier=1, min=0.1, max=5), - retry=retry_if_exception_type((ConnectionError, requests.RequestException)), - reraise=True, - ) - def wait_for_healthy(self): - response = self.health() - response = json.loads(response.content) - self.version = response.get("version", "unknown") - logging.debug("StatusBackend is healthy") - - def health(self): - return self.api_request("health", data=[], url=self.base_url, quiet=True) - - def initialize(self): - if Config.logout: - logging.warning("automatically logging out before InitializeApplication") - try: - self.logout() - logging.debug("successfully logged out") - except Exception: - logging.debug("failed to log out") - pass - - method = "InitializeApplication" - data = { - "dataDir": self.data_dir, - "logEnabled": True, - "logLevel": self.logLevel, - "apiLoggingEnabled": True, - "wakuFleetsConfigFilePath": Config.waku_fleets_config, - "pushFleetsConfigFilePath": Config.push_fleets_config, - "mediaServerAddress": f"""{"0.0.0.0" if self.container else "127.0.0.1"}:{constants.STATUS_MEDIA_SERVER_PORT if self.container else 0}""", - "mediaServerAdvertizeHost": "localhost" if self.container else "", - "mediaServerAdvertizePort": self.container.media_server_port if self.container else 0, - } - - return self.api_request_json(method, data) - - def _set_networks(self, data, **kwargs): - self.network_id = kwargs.get("network_id", ANVIL_NETWORK_ID) - anvil_network = { - "chainID": self.network_id, - "chainName": "Anvil", - "rpcProviders": [ - { - "chainId": self.network_id, - "name": "Anvil Direct", - "url": "http://anvil:8545", - "enableRpsLimiter": False, - "type": "embedded-direct", - "enabled": True, - "authType": "no-auth", - } - ], - "shortName": "eth", - "nativeCurrencyName": "Ether", - "nativeCurrencySymbol": "ETH", - "nativeCurrencyDecimals": 18, - "isTest": False, - "layer": 1, - "enabled": True, - "isActive": True, - "isDeactivatable": False, - } - anvil_network = self._set_token_overrides(anvil_network, kwargs.get("token_overrides", [])) - - data["testNetworksEnabled"] = False - data["networkId"] = self.network_id - data["networksOverride"] = [anvil_network] - - def _set_proxy_credentials(self, data): - if "STATUS_BUILD_PROXY_USER" not in os.environ: - return data - - user = os.environ["STATUS_BUILD_PROXY_USER"] - password = os.environ["STATUS_BUILD_PROXY_PASSWORD"] - - data["StatusProxyMarketUser"] = user - data["StatusProxyMarketPassword"] = password - data["StatusProxyBlockchainUser"] = user - data["StatusProxyBlockchainPassword"] = password - - data["StatusProxyEnabled"] = True - data["StatusProxyStageName"] = "test" - return data - - def _set_wallet_secrets(self, data): - if "STATUS_BUILD_INFURA_TOKEN" in os.environ: - data["infuraToken"] = os.environ["STATUS_BUILD_INFURA_TOKEN"] - if "STATUS_BUILD_INFURA_SECRET" in os.environ: - data["infuraSecret"] = os.environ["STATUS_BUILD_INFURA_SECRET"] - if "STATUS_BUILD_POKT_TOKEN" in os.environ: - data["poktToken"] = os.environ["STATUS_BUILD_POKT_TOKEN"] - return data - - def _set_token_overrides(self, network, token_overrides): - if not token_overrides: - return network - - network["TokenOverrides"] = token_overrides - return network - - def _set_multicall_overrides(self, data, kwargs): - multicall_contract_address = kwargs.get("multicall_contract_address", None) - if not multicall_contract_address: - return data - - data["multicallOverrides"] = {self.network_id: multicall_contract_address} - return data - - def extract_data(self, path: str): - if self.container: - return self.container.extract_data(path) - - if not os.path.exists(path): - return None - - return path - - def import_data(self, src_path: str, dest_path: str): - """ - Import a file from the host (src_path) into the container at dest_path. - If not running in a container, just copy the file locally. - """ - if self.container: - self.container.import_data(src_path, dest_path) - return - - # Not running in a container, just copy the file locally - if not os.path.exists(src_path): - raise FileNotFoundError(f"Source path '{src_path}' does not exist.") - - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - with open(src_path, "rb") as src, open(dest_path, "wb") as dst: - dst.write(src.read()) - - def _set_display_name(self, **kwargs): - self.display_name = kwargs.get("display_name", fake.profile_name()) - - def _create_account_request(self, password: str, **kwargs): - self.password = password - data = { - "rootDataDir": self.data_dir, - "kdfIterations": 256000, - # Profile config - "displayName": self.display_name, - "password": self.password, - "customizationColor": kwargs.get("customizationColor", "primary"), - # Logs config - "logEnabled": True, - "logToStderr": True, - "logLevel": self.logLevel, - # Waku config - "wakuV2LightClient": kwargs.get("waku_light_client", False), - "wakuV2Fleet": Config.waku_fleet, - # Connector config - "apiConfig": { - "apiModules": "connector", - "connectorEnabled": kwargs.get("connector_enabled", False), - "httpEnabled": False, - "httpHost": "0.0.0.0", - "httpPort": 0, - "wsEnabled": True, - "wsHost": "0.0.0.0", - "wsPort": constants.STATUS_CONNECTOR_WS_PORT, - }, - "thirdpartyServicesEnabled": True, - } - if not Config.disable_override_networks: - self._set_networks(data, **kwargs) - - data = self._set_proxy_credentials(data) - data = self._set_wallet_secrets(data) - data = self._set_multicall_overrides(data, kwargs) - return data - - def create_account_and_login(self, password: str, **kwargs): - self._set_display_name(**kwargs) - method = "CreateAccountAndLogin" - data = self._create_account_request(password=password, **kwargs) - return self.api_request_json(method, data) - - def restore_account_and_login(self, user=user_1, **kwargs): - self._set_display_name(**kwargs) - method = "RestoreAccountAndLogin" - data = self._create_account_request(password=user.password, **kwargs) - data["mnemonic"] = user.passphrase - return self.api_request_json(method, data) - - def login(self, key_uid, password: str, kdf_iterations=256000): - self.password = password - method = "LoginAccount" - data = { - "password": self.password, - "keyUid": key_uid, - "kdfIterations": kdf_iterations, - } - data = self._set_proxy_credentials(data) - data = self._set_wallet_secrets(data) - return self.api_request_json(method, data) - - def logout(self, **kwargs): - method = "Logout" - return self.api_request_json(method, {}, **kwargs) - - def wait_for_login(self): - signal = self.wait_for_signal(SignalType.NODE_LOGIN.value) - if "error" in signal["event"]: - error_details = signal["event"]["error"] - assert not error_details, f"Unexpected error during login: {error_details}" - self.node_login_event = signal - logging.debug(f"Node login event: {self.node_login_event}") - self.public_key = self.node_login_event.get("event", {}).get("settings", {}).get("public-key") - self.mnemonic = self.node_login_event.get("event", {}).get("settings", {}).get("mnemonic") - self.key_uid = self.node_login_event.get("event", {}).get("account", {}).get("key-uid") - return signal - - def wait_for_messages(self, timeout: int | None = 20): - return self.wait_for_signal(SignalType.MESSAGES_NEW, timeout) - - def container_pause(self): - if not self.container: - raise RuntimeError("Container is not initialized.") - self.container.pause() - - def container_unpause(self): - if not self.container: - raise RuntimeError("Container is not initialized.") - self.container.unpause() - - def container_exec(self, command): - if not self.container: - raise RuntimeError("Container is not initialized.") - return self.container.exec(command) - - def compressed_public_key(self): - if not self.public_key: - return "" - return keys.compress_public_key(self.public_key) - - @retry(stop=stop_after_delay(10), wait=wait_fixed(0.1), reraise=True) - def change_container_ip(self, new_ipv4=None, new_ipv6=None): - if not self.container: - raise RuntimeError("Container is not initialized.") - self.container.change_ip(new_ipv4, new_ipv6) - - def wait_for_online(self, timeout=10): - start_time = time.time() - while time.time() - start_time <= timeout: - response = self.wakuext_service.peers() - if len(response.keys()) == 0: - time.sleep(0.5) - continue - logging.info(f"StatusBackend is online after {time.time() - start_time} seconds") - return - raise TimeoutError(f"StatusBackend was not online after {timeout} seconds") - - def get_connection_string_for_bootstrapping_another_device(self, message_sync_enabled=False): - method = "GetConnectionStringForBootstrappingAnotherDevice" - data = { - "senderConfig": { - "keystorePath": os.path.join(self.data_dir, "keystore", self.key_uid), - "deviceType": "macos", - "keyUID": self.key_uid, - "password": self.password, - "chatKey": "", - "messageSyncingEnabled": message_sync_enabled, - }, - "serverConfig": { - "timeout": 5 * 60 * 1000, - }, - } - response = self.api_request(method, data) - return response.content.decode() - - def input_connection_string_for_bootstrapping(self, connection_string): - method = "InputConnectionStringForBootstrappingV2" - # Empty user - data = { - "connectionString": connection_string, - "receiverClientConfig": { - "receiverConfig": {"createAccount": self._create_account_request(password="")}, - "clientConfig": {}, - }, - } - return self.api_request_json(method, data) - - def get_connection_string_for_being_bootstrapped(self): - method = "GetConnectionStringForBeingBootstrapped" - data = { - "receiverConfig": { - "createAccount": self._create_account_request(password=""), - "deviceType": "macos", - }, - "serverConfig": { - "timeout": 5 * 60 * 1000, - }, - } - response = self.api_request(method, data) - return response.content.decode() - - def input_connection_string_for_bootstrapping_another_device(self, connection_string): - method = "InputConnectionStringForBootstrappingAnotherDeviceV2" - data = { - "connectionString": connection_string, - "senderClientConfig": { - "senderConfig": { - "keystorePath": os.path.join(self.data_dir, "keystore", self.key_uid), - "deviceType": "macos", - "keyUID": self.key_uid, - "password": self.password, - "chatKey": "", - }, - "clientConfig": {}, - }, - } - return self.api_request_json(method, data) - - def gather_metrics(self): - if not self.container: - raise RuntimeError("Gathering metrics is only supported when running status-backend in a Docker container") - - # Stop both monitoring threads and get independent arrays - container_stats = self.container.stop_performance_monitoring() - go_metrics = self.expvar_client.stop_monitoring() - - # Create PerformanceMetrics with independent arrays - return StatusGoMetrics( - container_stats=container_stats, - go_metrics=go_metrics, - events=self.events, - version=self.version, - ) - - def start_performance_monitoring(self): - """Start performance monitoring with independent threads""" - if not self.container: - raise RuntimeError("Performance monitoring is only supported when running status-backend in a Docker container") - - self.container.start_performance_monitoring() - self.expvar_client.start_monitoring() - - def free_os_memory(self): - url = f"{self.base_url}/statusgo/debug/FreeOSMemory" - requests.post(url) - - def change_database_password(self, old_password, new_password): - method = "ChangeDatabasePasswordV2" - data = { - "keyUid": self.key_uid, - "oldPassword": old_password, - "newPassword": new_password, - } - return self.api_request_json(method, data) - - def image_server_tls_cert(self): - method = "ImageServerTLSCert" - response = self.api_request(method, {}) - return response.content.decode("utf-8") - - def serialize_legacy_key(self, key): - method = "SerializeLegacyKey" - # Use client.post directly, because this method is old and has json-incompatible arguments - response = self.client.post(self.method_url(method), data=key) - return response.content.decode() - - def start_with_account(self, display_name: str, password: str, identity_image_path: str = "", **kwargs): - response = self.initialize() - if response is None: - response = {} - account_created = False - for account in response.get("accounts", []) or []: - if account["name"] == display_name: - self.login(account["key-uid"], password=password) - break - else: - print(f"Account '{display_name}' not found, creating...") - self.create_account_and_login(password=password, display_name=display_name) - account_created = True - - try: - self.wait_for_login() - self.wakuext_service.start_messenger() - self.wallet_service.start_wallet() - except Exception as e: - if "node is already running" not in str(e): - raise e - - if account_created: - self.multiaccounts_service.store_identity_image(self.key_uid, identity_image_path, 0, 0, 1024, 1024) - - def generate_profile_qr_code(self): - bot_url = self.wakuext_service.share_user_url_with_data(self.public_key) - print(f"--- URL: {bot_url}") - print(f"--- Public Key: {self.public_key}") - - img = qrcode.make(bot_url) - img.save("qr_code.png") # type: ignore diff --git a/clients/statusgo_container.py b/clients/statusgo_container.py deleted file mode 100644 index ca4e4f1..0000000 --- a/clients/statusgo_container.py +++ /dev/null @@ -1,479 +0,0 @@ -import io -import logging -import os -import random -import tarfile -import tempfile -import threading - -import docker -import docker.errors -from docker.errors import APIError - -import resources.constants as constants -from clients.metrics import ContainerStats -from utils.config import Config - -DATA_DIR = "/usr/status-user" - - -class StatusGoContainer: - all_containers = [] - container = None - - def __init__(self, cmd, ports=None, privileged=False, container_name_suffix=""): - if ports is None: - ports = {} - - # Initialize stop event for monitoring thread - self._stop_monitoring = threading.Event() - self.health_monitor = None - self._stop_perf_monitoring = threading.Event() - self.perf_monitor = None - - # Initialize performance metrics container - self.stats = list[ContainerStats]() - - # Prepare image and container name - # NOTE: This part needs some love. - # There's magic with `docker_project_name`, `docker_image` and `identifier` variables. - docker_project_name = Config.docker_project_name - self.network_name = f"{docker_project_name}_default" - git_commit = os.popen("git rev-parse --short HEAD").read().strip() - identifier = os.environ.get("BUILD_ID") if os.environ.get("CI") else git_commit - image_name = Config.docker_image or f"statusgo-{identifier}:latest" - self.container_name = f"{docker_project_name}-{identifier}{container_name_suffix}" - coverage_path = Config.codecov_dir if Config.codecov_dir else os.path.abspath("./coverage/binary") - - # Run the container - logging.debug(f"Creating status-go container from image '{image_name}'") - - container_args = { - "image": image_name, - "detach": True, - "privileged": privileged, - "name": self.container_name, - "labels": {"com.docker.compose.project": docker_project_name}, - "environment": { - "GOCOVERDIR": "/coverage/binary", - "SCAN_WAKU_FLEET": self.get_waku_fleet_scan_command(), - }, - "volumes": { - coverage_path: { - "bind": "/coverage/binary", - "mode": "rw", - } - }, - "extra_hosts": { - "host.docker.internal": "host-gateway", - }, - "command": cmd, - "ports": ports, - "stop_signal": "SIGINT", - "network": self.network_name, - } - - if "FUNCTIONAL_TESTS_DOCKER_UID" in os.environ: - container_args["user"] = os.environ["FUNCTIONAL_TESTS_DOCKER_UID"] - - self.docker_client = docker.from_env() - - try: - self.docker_client.images.get(image_name) - except docker.errors.ImageNotFound: - raise RuntimeError(f"Docker image '{image_name}' not found") - - self.container = self.docker_client.containers.run(**container_args) - StatusGoContainer.all_containers.append(self) - - logging.debug(f"Container {self.container.name} created. ID = {self.container.id}") - - def get_waku_fleet_scan_command(self): - """Returns the command string for scanning Waku fleet and generating config""" - - # Known node names from docker compose - bootstrap_nodes = "boot-1" - static_nodes = "boot-1" # Add bootnode, otherwise metadata exchange doesn't happen, and Waku light mode doesn't work - store_nodes = "store" - - return ( - "python3 /usr/local/bin/scan_waku_fleet.py " - f"--fleet-name {Config.waku_fleet} " - f"--cluster-id 16 " # Cluster ID matches docker-compose.waku.yml - f"--bootstrap-nodes {bootstrap_nodes} " - f"--store-nodes {store_nodes} " - f"--static-nodes {static_nodes} " - f"--output {Config.waku_fleets_config}" - ) - - def __del__(self): - self.stop() - - def data_dir(self): - return DATA_DIR - - def id(self): - return self.container.id if self.container else "" - - def short_id(self): - return self.container.id[:8] if self.container else "" - - def name(self): - return self.container.name if self.container else "" - - def _check_container_health(self): - """Check if container is healthy""" - if not self.container: - raise RuntimeError("Container is not initialized") - - self.container.reload() - if self.container.status != "running": - logs = self.container.logs().decode("utf-8").splitlines()[-10:] - logs = "\n".join(logs) - raise RuntimeError(f"Container is not running. Status: {self.container.status}. Logs (last 10 lines):\n{logs}") - return True - - def start_health_monitoring(self): - """Start background health monitoring thread""" - - def monitor(): - while not self._stop_monitoring.is_set(): - try: - self._check_container_health() - # Wait for 5 seconds or until stop event is set - self._stop_monitoring.wait(timeout=1) - except Exception as e: - logging.error(f"Container health check failed: {e}") - raise e # This will kill the thread and fail the test - - self._stop_monitoring.clear() # Reset the event - self.health_monitor = threading.Thread(target=monitor, daemon=True) - self.health_monitor.start() - - def start_performance_monitoring(self): - """Start independent container performance monitoring thread""" - # Reset metrics storage - self.container_stats = [] - self._stop_perf_monitoring = threading.Event() - - def monitor_performance(): - stats_stream = self.docker_client.api.stats(self.id(), decode=True, stream=True) - prev_stat = None - - for stat in stats_stream: - # Create ContainerStats with only container data - container_stats = ContainerStats(stat, prev_stat, go_memory_stats=None) - self.container_stats.append(container_stats) - - # Store current stat as previous for the next iteration - prev_stat = stat - - if self._stop_perf_monitoring.is_set(): - break - - logging.debug(f"Performance monitoring stopped for container {self.name()}") - - self._stop_perf_monitoring.clear() - self.perf_monitor = threading.Thread(target=monitor_performance, daemon=True) - self.perf_monitor.start() - logging.info(f"Started performance monitoring for container {self.name()}") - - def stop_performance_monitoring(self): - """Stop the performance monitoring thread and return the collected metrics""" - self._stop_perf_monitoring.set() # Signal the thread to stop - if not self.perf_monitor or not self.perf_monitor.is_alive(): - return [] - - self.perf_monitor.join(timeout=10) - if self.perf_monitor.is_alive(): - logging.warning("Performance monitoring thread didn't stop gracefully") - - return self.container_stats - - def stop_health_monitoring(self): - """Stop the health monitoring thread""" - self._stop_monitoring.set() # Signal the thread to stop - if not self.health_monitor or not self.health_monitor.is_alive(): - return - self.health_monitor.join(timeout=10) - if self.health_monitor.is_alive(): - logging.warning("Health monitoring thread didn't stop gracefully") - - def shutdown(self, log_sufix=""): - """ - Stops, saves logs, and removes a container with error handling. - Args: - log_sufix: Optional string for logging context - """ - if not self.container: - return - - container_id = self.short_id() - self.stop() - self.save_logs(log_sufix) - self.remove() - logging.debug(f"Container '{container_id}' shutdown finished") - - def stop(self): - """Stop the container and monitoring""" - self.stop_health_monitoring() # Stop health monitoring first - if hasattr(self, "_stop_perf_monitoring"): - self.stop_performance_monitoring() # Stop performance monitoring if running - if self.container: - logging.debug(f"Stopping container {self.container.name}...") - self.container.stop(timeout=10) - logging.debug(f"Container {self.container.name} stopped.") - - def remove(self): - """Remove the container""" - if self.container: - name = self.container.name - logging.debug(f"Removing container {name}...") - self.container.remove() - self.container = None - logging.debug(f"Container {name} removed.") - - def pause(self): - if not self.container: - raise RuntimeError("Container is not initialized.") - self.container.pause() - logging.info(f"Container {self.container.name} paused.") - - def unpause(self): - if not self.container: - raise RuntimeError("Container is not initialized.") - self.container.unpause() - logging.info(f"Container {self.container.name} unpaused.") - - def exec(self, command): - if not self.container: - raise RuntimeError("Container is not initialized.") - try: - exec_result = self.container.exec_run(cmd=["sh", "-c", command], stdout=True, stderr=True, tty=False) - if exec_result.exit_code != 0: - raise RuntimeError(f"Failed to execute command in container {self.container.id}:\n" f"OUTPUT: {exec_result.output.decode().strip()}") - return exec_result.output.decode().strip() - except APIError as e: - raise RuntimeError(f"API error during container execution: {str(e)}") from e - - def extract_data(self, path: str): - if not self.container: - raise RuntimeError("Container is not initialized.") - - try: - stream, _ = self.container.get_archive(path) - except docker.errors.NotFound: - logging.error(f"Path '{path}' not found in container {self.container.name}.") - return None - - temp_dir = tempfile.mkdtemp() - tar_bytes = io.BytesIO(b"".join(stream)) - - with tarfile.open(fileobj=tar_bytes) as tar: - tar.extractall(path=temp_dir) - # If the tar contains a single file, return the path to that file - # Otherwise it's a directory, just return temp_dir. - if len(tar.getmembers()) == 1: - return os.path.join(temp_dir, tar.getmembers()[0].name) - - return temp_dir - - def import_data(self, src_path: str, dest_path: str): - """ - Copy data from the host (src_path) into the container at dest_path. - """ - if not self.container: - raise RuntimeError("Container is not initialized.") - - if not os.path.exists(src_path): - raise FileNotFoundError(f"Source path '{src_path}' does not exist.") - - # Create a tar archive of the source path - tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode="w") as tar: - arcname = os.path.basename(src_path) - tar.add(src_path, arcname=arcname) - tar_stream.seek(0) - - # Put the archive into the container at the destination path - try: - # Ensure destination directory exists in the container - response = self.container.exec_run(cmd=["mkdir", "-p", dest_path]) - assert response.exit_code == 0, f"Failed to ensure directory exists: {response.output.decode().strip()}" - success = self.container.put_archive(dest_path, tar_stream.getvalue()) - assert success, f"Failed to put archive to {dest_path} in container {self.container.name}" - except Exception as e: - logging.error(f"Failed to import data to container: {e}") - raise - - def get_name(self): - return self.container.name if self.container else None - - def save_logs(self, log_sufix="test"): - if not self.container: - raise RuntimeError("Container is not initialized.") - if Config.logs_dir == "": - logging.debug("Save container logs skipped") - return - - os.makedirs(Config.logs_dir, exist_ok=True) - - file_path = os.path.join(Config.logs_dir, f"container_{log_sufix}_{self.short_id()}.log") - logging.info(f"Saving logs to {file_path}") - - with open(file_path, "wb") as f: - logs = self.container.logs() - f.write(logs) - - @staticmethod - def acquire_port(): - host_port = random.choice(Config.status_backend_port_range) - Config.status_backend_port_range.remove(host_port) - return host_port - - def connect_to_bridge_network(self): - if not self.container: - return - - networks_attached = set(self.container.attrs.get("NetworkSettings", {}).get("Networks", {}).keys() or []) - if "bridge" in networks_attached: - return - try: - bridge_net = self.docker_client.networks.get("bridge") - bridge_net.connect(self.container) - logging.info(f"Connected container {self.container.name} to bridge network") - except docker.errors.APIError as e: - if "already exists" in str(e).lower(): - # Not an error - logging.debug(f"Bridge connection already exists for {self.container.name}") - return - # Otherwise re-raise the exception - raise e - - -class PushNotificationServerContainer(StatusGoContainer): - def __init__(self, identity, gorush_port): - entrypoint = [ - "push-notification-server", - "--identity", - identity, - "--gorush-url", - f"http://host.docker.internal:{gorush_port}", - "--data-dir", - DATA_DIR, - "--log-level", - "DEBUG", - "--waku-fleet-config", - Config.waku_fleets_config, - "--waku-fleet", - Config.waku_fleet, - ] - super().__init__(entrypoint, container_name_suffix=f"-push-notification-server-{gorush_port}") - - -class StatusBackendContainer(StatusGoContainer): - def __init__(self, privileged=False, ipv6=False, **kwargs): - connector_enabled = kwargs.get("connector_enabled", False) - - host_port = StatusGoContainer.acquire_port() - connector_ws_port = StatusGoContainer.acquire_port() if connector_enabled else 0 - self.media_server_port = StatusGoContainer.acquire_port() - - container_port = 3333 - entrypoint = [ - "status-backend", - "--address", - f"0.0.0.0:{container_port}" if not ipv6 else f"[::]:{container_port}", - "--pprof", - "true" if kwargs.get("pprof_enabled", False) else "false", - ] - - self.ipv6 = ipv6 - - if ipv6: - ports = { - f"{container_port}/tcp": [ - {"HostIp": "::", "HostPort": str(host_port)}, - ], - f"{constants.STATUS_MEDIA_SERVER_PORT}/tcp": [ - {"HostIp": "::", "HostPort": str(self.media_server_port)}, - ], - } - if connector_enabled: - ports[f"{constants.STATUS_CONNECTOR_WS_PORT}/tcp"] = [{"HostIp": "::", "HostPort": str(connector_ws_port)}] - - self.url = f"http://[::1]:{host_port}" - self.connector_ws_url = f"ws://[::1]:{connector_ws_port}" - else: - ports = { - f"{container_port}/tcp": str(host_port), - f"{constants.STATUS_MEDIA_SERVER_PORT}/tcp": str(self.media_server_port), - } - if connector_enabled: - ports[f"{constants.STATUS_CONNECTOR_WS_PORT}/tcp"] = str(connector_ws_port) - self.url = f"http://127.0.0.1:{host_port}" - self.connector_ws_url = f"ws://127.0.0.1:{connector_ws_port}" - - super().__init__(entrypoint, ports, privileged, container_name_suffix=f"-status-backend-{host_port}") - - bridge_network = kwargs.get("bridge_network", False) - if bridge_network: - self.connect_to_bridge_network() - - def _change_ip(self, new_ipv4=None, new_ipv6=None): - if not self.container: - raise RuntimeError("Container is not initialized.") - - # Get the network details - network = self.docker_client.networks.get(self.network_name) - - # Ensure network has explicitly configured subnets - ipam_config = network.attrs.get("IPAM", {}).get("Config", []) - if not ipam_config: - raise RuntimeError("Network does not have a user-defined subnet, cannot assign a custom IP.") - - self.container.reload() - container_info = self.container.attrs["NetworkSettings"]["Networks"].get(self.network_name, {}) - current_ipv4 = container_info.get("IPAddress", "Unknown") - current_ipv6 = container_info.get("GlobalIPv6Address", "Unknown") - - logging.info(f"Current IPs for {self.container.name} - IPv4: {current_ipv4}, IPv6: {current_ipv6}") - - # Generate new IPs based on mode - for config in ipam_config: - subnet = config.get("Subnet") - - if self.ipv6 and ":" in subnet and not new_ipv6: # IPv6 Subnet - base_ipv6 = subnet.rstrip("::/64") - new_ipv6 = f"{base_ipv6}::{random.randint(1, 9999):x}:{random.randint(1, 9999):x}" - logging.info(f"Generated new IPv6: {new_ipv6}") - - elif not self.ipv6 and "." in subnet and not new_ipv4: # IPv4 Subnet - new_ipv4 = subnet.rsplit(".", 1)[0] + f".{random.randint(2, 254)}" - logging.info(f"Generated new IPv4: {new_ipv4}") - - # Disconnect and reconnect with only the needed IP type - network.disconnect(self.container) - if self.ipv6: - network.connect(self.container, ipv6_address=new_ipv6) - else: - network.connect(self.container, ipv4_address=new_ipv4) - - self.container.reload() - updated_info = self.container.attrs["NetworkSettings"]["Networks"].get(self.network_name, {}) - updated_ipv4 = updated_info.get("IPAddress", "Unknown") - updated_ipv6 = updated_info.get("GlobalIPv6Address", "Unknown") - - if self.ipv6 and current_ipv6 == updated_ipv6: - raise RuntimeError("IPV6 is the same after network reconnect") - if not self.ipv6 and current_ipv4 == updated_ipv4: - raise RuntimeError("IPV4 is the same after network reconnect") - - logging.info(f"Changed container {self.container.name} IPs - New IPv4: {updated_ipv4}, New IPv6: {updated_ipv6}") - - def change_ip(self, new_ipv4=None, new_ipv6=None): - try: - logging.info(f"Trying to change container {self.container_name} IPs (IPv6 Mode: {self.ipv6})") - self._change_ip(new_ipv4, new_ipv6) - except Exception as e: - raise RuntimeError(f"Failed to change container IP: {e}") diff --git a/config.yaml b/config.yaml index cca55d3..5e23f54 100644 --- a/config.yaml +++ b/config.yaml @@ -4,19 +4,14 @@ postgres: messages: "raw_messages" community: "raw_community_info" members: "raw_community_activity" - -status_app: - bot_name: null # Optional. Can be specified in .env - read README.md - channels: - - https://status.app/c/G6EAAMSs5eYUrSjkDriqGHx1OITK3bd8aUlQKA9M5Mg08uTbwYKNMVxLXxDGfzde3Ub9OeDeNCmVTbP-vZs-rsCtWIUKDBBWUBXrEaGpJQ5Kaj0o4pYlcJ0iLlnP-MQRxCRwy3pE3JOiMxYYIyb4WtmaksZHTHQKCLUc14iNpWoidEbVeeO2g931cXu8Lns3Bw==#zQ3shsFYujbDQdRhSKS9RHuGCwxHQ1WLkNYvGPRksf4ebDWFW - url: - port: 8080 - is_https: false - backend_params: - logLevel: "INFO" - data_dir: "./data-dir" - sleep: # Values must be in MINUTES upload: 15 # 15 min download: 1440 # 1 day +# Parameters based on Account class +bot_params: + # localhost - to run code locally with Status Backend Dockerfile + # status-backend - to run code with docker-compose.yaml + domain: "status-backend" + port: 8080 + is_secure: false diff --git a/constants.py b/constants.py deleted file mode 100644 index ce88f30..0000000 --- a/constants.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import yaml -from dotenv import load_dotenv as __load_dotenv - -CREDENTIALS_PATH = os.path.join(os.path.dirname(__file__), "accounts") -UPLOAD_PATH = os.path.join(os.path.join(os.path.dirname(__file__), "uploads")) - -with open(os.path.join(os.path.dirname(__file__), "config.yaml"), "r") as f: - CONFIG: dict = yaml.safe_load(f) - -__load_dotenv() - -# Make bot_name optional in config.yaml -# You can use .env STATUS_USERNAME as an alternative -__bot_name = CONFIG["status_app"].get("bot_name") -if not __bot_name: - __bot_name = "" - -if len(__bot_name) == 0 and os.environ.get("STATUS_USERNAME"): - CONFIG["status_app"]["bot_name"] = os.environ.get("STATUS_USERNAME") - -STATUS_BACKEND_PARAMS = { - **CONFIG["status_app"]["backend_params"], - "url": f"http{'s' if CONFIG['status_app']['url']['is_https'] else ''}://{os.environ.get('STATUS_BACKEND_BASE_URL', 'localhost')}:{CONFIG['status_app']['url']['port']}" -} diff --git a/create_account.py b/create_account.py deleted file mode 100644 index 907f1b7..0000000 --- a/create_account.py +++ /dev/null @@ -1,55 +0,0 @@ -from clients.status_backend import StatusBackend -import os, json, argparse, logging -import constants, data_utils - -def main(username: str, password: str, logger: logging.Logger): - os.makedirs(constants.CREDENTIALS_PATH, exist_ok=True) - file_path = os.path.join(constants.CREDENTIALS_PATH, f"{username}.json") - if os.path.exists(file_path): - logger.info(f"There is a JSON file for {username}. Skipping account creation.") - return - - backend = StatusBackend(**constants.STATUS_BACKEND_PARAMS) - info = backend.initialize() - - params = { - "display_name": username, - "password": password - } - logger.info(f"Creating account for {username}") - backend.create_account_and_login(**params) - info = backend.wait_for_login() - logger.info(f"Created account for {username}!") - data = { - "event": info["event"], - "created_at": info["timestamp"], - "credentials": params - } - - with open(file_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=4, ensure_ascii=False) - - logger.info(f"JSON data for {username} saved in {file_path}") - backend.logout() - logger.info("Logged out of Status App.") - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description="Login script") - parser.add_argument( - "-u", "--username", - help="Status App login username" - ) - parser.add_argument( - "-p", "--password", - help="Status App local password" - ) - args = parser.parse_args() - - params = { - "username": os.environ.get("STATUS_USERNAME") if not args.username else args.username, - "password": os.environ.get("STATUS_PASSWORD") if not args.password else args.password, - "logger": data_utils.get_logger("create-account") - } - - main(**params) diff --git a/data_utils.py b/data_utils.py deleted file mode 100644 index 854d455..0000000 --- a/data_utils.py +++ /dev/null @@ -1,270 +0,0 @@ -""" -Function that simplify the Status Backend data extraction process -""" -from clients.status_backend import StatusBackend -from typing import Optional -import os, json, datetime, json, logging -import constants - -def get_logger(caller: str) -> logging.Logger: - """ - Custom logger that is used to monitor the upload and download processes - - Params: - - `caller` - if the file is called in `download.py` or `upload.py` - - Output: - - Initialized logger - """ - # upload and download loggers should have different names - # if we have to store the output in a `.log` file - logger = logging.getLogger(f"status-monitoring-{caller}") - logger.propagate = False - if not logger.handlers: - logger.setLevel(logging.INFO) - - handler = logging.StreamHandler() - - formatter = logging.Formatter( - f"[%(asctime)s] [%(levelname)s]\t[{caller}]\t%(message)s", - datefmt="%Y-%m-%d %H:%M:%S" - ) - - handler.setFormatter(formatter) - logger.addHandler(handler) - - return logger - -def login(backend: StatusBackend, username: str, is_chat: bool = True) -> dict: - """ - Login to Status Backend with the given username. - NOTE: Account must already exist! - - Parameters: - - `backend` - the `StatusBackend` - - `username` - the username of the account - - `is_chat` - if `True` then Logos Messaging will be launched - - Output: - - the account information - """ - file_path = os.path.join(constants.CREDENTIALS_PATH, f"{username}.json") - if not os.path.exists(file_path): - raise Exception(f"Please make sure you create an account with file 'create_account.py' before running this file") - - with open(file_path, "r", encoding="utf-8") as f: - account_info: dict = json.load(f) - - params = { - "key_uid": account_info["event"]["account"]["key-uid"], - "password": account_info["credentials"]["password"] - } - backend.initialize() - backend.login(**params) - backend.wait_for_login() - - if is_chat: - backend.wakuext_service.start_messenger() - - output = { - "account_name": account_info["event"]["account"]["name"], - "account_key_uid": params["key_uid"], - "account_key": account_info["event"]["settings"]["compressedKey"] - } - return output - - - -def get_community_info(backend: StatusBackend, url: str) -> dict: - """ - Get community information for the given URL. - - **Note**: This function will not work if the URL is to a specific channel in a community. - - Parameters: - - `backend` - the `StatusBackend` - - `url` - the URL of the community - - Output: - - community and channel information - """ - response = backend.sharedurls_service.parse_shared_url(url) - community_info: dict = backend.wakuext_service.fetch_community(response["community"]["communityId"]) - if not community_info: - raise Exception(f"No community info found for {url}") - - to_datetime = lambda key: datetime.datetime.fromtimestamp(community_info[key]) if key in community_info else None - extract_timestamp = datetime.datetime.now() - data = { - "community_id": community_info["id"], - "community_name": community_info["name"], - "url": url, - "verified": community_info["verified"], - "description": community_info["description"], - "dialog": community_info["introMessage"], - "leaving_message": community_info["outroMessage"], - "tags": community_info["tags"], - "is_member": community_info["isMember"], - "joined_timestamp": to_datetime("joinedAt"), - "requested_timestamp": to_datetime("requestedToJoinAt"), - "encrypted": community_info["encrypted"], - "members": { - "total": len(community_info["members"].keys()), - "info": [ - { - "member_id": member_id, - "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None, - "extract_timestamp": extract_timestamp - } - for member_id, info in community_info["members"].items() - ] - }, - "channels": [ - { - "community_id": community_info["id"], - "channel_id": chat_info["id"], - "chat_id": community_info["id"] + chat_info["id"], - "category_id": chat_info["categoryID"] if len(chat_info["categoryID"]) > 0 else None, - "channel_name": chat_info["name"], - "description": chat_info["description"], - } - for chat_info in list(community_info["chats"].values()) - ] - } - - return data - - -def get_contacts(backend: StatusBackend) -> list[dict]: - """ - Get all of the contacts that the bot has. - NOTE: You have to be logged in to Status app already! - - Parameters: - - `backend` - the `StatusBackend` - - Output: - - the contacts that the bot has - """ - info = [ - { - "id": user["id"], - "name": user["displayName"] if len(user["displayName"]) > 0 else None, - "chat_key": user["compressedKey"], - "emojis": "; ".join(user["emojiHash"]) - } - for user in backend.wakuext_service.get_contacts() - ] - return info - - - -def save_messages( - backend: StatusBackend, - chat_id: str, - folder: str, - community_info: dict, - start_timestamp: Optional[datetime.datetime] = None, - end_timestamp: Optional[datetime.datetime] = None, - pagination: int = 500, - batch_size: int = 10_00 - ): - """ - Get all of the mesages from a chat. - NOTE: You have to be logged in to Status app already! - - Params: - - `chat_id` - the chat id that will be scraped - - `folder` - where the batched data will be saved - - `start_timestamp` - the start extraction timestamp. Messages before this timestamp will be ignored. - - `end_timestamp` - the end extraction timestamp. Messages after this timestamp will be ignored. - - `pagination` - how many results to get per `.chat_messages` call - - `batch_size` - the number of messages that will be turned into a batch - """ - def save_batch(messages: list[str], folder: str): - timestamp = datetime.datetime.now().timestamp() - - file_path = os.path.join(folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".json") - data = { - "metadata": { - "file_path": file_path, - "created_at": timestamp, - "total_messages": len(messages), - "earliest_msg_timestamp": min(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"], - "latest_msg_timestamp": max(messages, key=lambda d: d["sent_timestamp"])["sent_timestamp"], - }, - "messages": messages, - } - with open(file_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=4, ensure_ascii=False) - - is_none = lambda value: isinstance(value, type(None)) - now = datetime.datetime.now() - if is_none(start_timestamp): - # `backend.wakuext_service.chat_messages` will only return you known / fetched messages for this channel. - # Without enabling community archives feature you can only fetch last 30 days (from waku store nodes). - start_timestamp = now - datetime.timedelta(days=30) - - if is_none(end_timestamp): - # t-1 is a safe assumption that messages will not be further modified / emojis added - end_timestamp = (now - datetime.timedelta(days=1)).replace(minute=0, second=0, hour=0, microsecond=0) - - if start_timestamp > end_timestamp: - raise Exception(f"Start timestamp ({start_timestamp}) cannot be greater than the end one ({end_timestamp})") - - os.makedirs(folder, exist_ok=True) - messages = [] - cursor = None - mappings = { - "id": "message_id", - "chatId": "chat_id", - "timestamp": "sent_timestamp", - "compressedKey": "from_key", - "emojiHash": "from_emojis", - "parsedText": "parsed_text", - "text": "markdown_text", - "editedAt": "edited_timestamp", - "links": "links" - } - finished = False - while not finished: - params = { - "chat_id": chat_id, - "limit": pagination - } - if cursor: - params["cursor"] = cursor - - chat: dict = backend.wakuext_service.chat_messages(**params) - for msg in chat["messages"]: - point = { - "community_id": community_info["community_id"], - "channel_id": community_info["chat_id"].replace(community_info["community_id"], ""), - "channel_category_id": community_info["category_id"], - **{target_key: msg[msg_key] for msg_key, target_key in mappings.items() if msg_key in msg}, - "extracted_at_timestamp": datetime.datetime.now().timestamp() - } - point["sent_timestamp"] /= 1_000 - timestamp = datetime.datetime.fromtimestamp(point["sent_timestamp"]) - if timestamp > end_timestamp: - continue - - if timestamp < start_timestamp: - finished = True - break - - messages.append(point) - - - if len(chat["cursor"]) > 0 and not finished: - cursor: str = chat["cursor"] - - finished = not bool(cursor) - - if len(messages) >= batch_size: - save_batch(messages, folder) - messages = [] - - - if messages: - save_batch(messages, folder) diff --git a/docker-compose.yaml b/docker-compose.yaml index 35da843..bdff4b5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,9 @@ --- services: backend: - image: harbor.status.im/bi/status-backend:dev + build: + context: https://github.com/status-im/status-go.git#develop + platform: linux/amd64 container_name: status-backend ports: - 8080:8080 @@ -25,7 +27,7 @@ services: - backend - database volumes: - - ./accounts:/app/accounts + - ./backups:/backups env_file: - .env networks: diff --git a/download.py b/download.py deleted file mode 100644 index ff0ea72..0000000 --- a/download.py +++ /dev/null @@ -1,54 +0,0 @@ -from clients.status_backend import StatusBackend -from typing import Any -import constants, data_utils -import os, datetime, pickle, time, logging - -def save_pkl(file_path: str, data: Any): - """ - Save the given data into a Pickel file - - Parameters: - - `file_path` - the path where the data will be saved - - `data` - the data that will be saved - """ - with open(file_path, "wb") as f: - pickle.dump(data, f) - -def run(logger: logging.Logger): - backend = StatusBackend(**constants.STATUS_BACKEND_PARAMS) - info = data_utils.login(backend, constants.CONFIG["status_app"]["bot_name"]) - logger.info(f"Logged in with account {constants.CONFIG['status_app']['bot_name']}") - - message_folder = os.path.join(constants.UPLOAD_PATH, "messages") - community_folder = os.path.join(constants.UPLOAD_PATH, "channel_info") - - for channel_url in constants.CONFIG["status_app"]["channels"]: - logger.info(f"Starting {channel_url}") - community = data_utils.get_community_info(backend, channel_url) - current_community_folder = os.path.join(community_folder, community["community_id"]) - os.makedirs(current_community_folder, exist_ok=True) - file_path = os.path.join(current_community_folder, str(datetime.datetime.now().timestamp()).replace(".", "") + ".pkl") - save_pkl(file_path, community) - - for channel in community["channels"]: - msg = f"Downloading messages from {community['community_name']} #{channel['channel_name']}" - logger.info(msg) - params = { - "backend": backend, - "chat_id": channel["chat_id"], - "folder": message_folder, - "community_info": channel - } - data_utils.save_messages(**params) - - backend.logout() - logger.info(f"Logged out of {constants.CONFIG['status_app']['bot_name']}") - -if __name__ == "__main__": - - logger = data_utils.get_logger("download") - while True: - run(logger) - seconds = 60 * constants.CONFIG["sleep"]["download"] - logger.info(f"Sleeping for {constants.CONFIG['sleep']['download']} minutes") - time.sleep(seconds) diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index 0e25ded..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ "${INIT_ACCOUNT}" = "true" ]; then - echo "Creating account" - python create_account.py $@ -fi - -python upload.py & python download.py diff --git a/monitor.py b/monitor.py new file mode 100644 index 0000000..533e0fb --- /dev/null +++ b/monitor.py @@ -0,0 +1,189 @@ +import datetime, os, pickle, yaml, time +import pandas as pd +from pathlib import Path +from dotenv import load_dotenv +from multiprocessing import Process +from collections.abc import Callable +# Manual file imports +from bot import Account, Logger +from postgres import Postgres + +def load_config(file_path: str) -> dict: + """ + Load the config file and the `.env` variables + + Parameter: + - `file_path` - the file path of the config yaml file. The `.env` variable must be in the same folder + + Output: + - The config variables and secret from `.env` + """ + with open(file_path, "r") as f: + config: dict = yaml.safe_load(f) + + env_file_path = os.path.join(os.path.dirname(file_path), ".env") + load_dotenv(env_file_path) + + config["env_vars"] = { + key: value + for key, value in os.environ.items() + if key.startswith(("POSTGRES_", "STATUS_")) + } + + return config + +def download(folder: str, config: dict): + """ + Download Status App messages / info from communities and store them in pickle files. + + Parameters: + - `folder` - the folder where the files will be created. Sub folders are automatically created + - `config` - the `load_config` configuration + """ + account = Account(**config.get("bot_params", {})) + available_accounts = [acc["display_name"] for acc in account.available_accounts] + + messages_folder = os.path.join(folder, "messages") + os.makedirs(messages_folder, exist_ok=True) + + community_info_folder = os.path.join(folder, "community") + os.makedirs(community_info_folder, exist_ok=True) + + members_info_folder = os.path.join(folder, "members") + os.makedirs(members_info_folder, exist_ok=True) + + prefix = "STATUS_" + params = { + key.replace(prefix, "").lower(): value + for key, value in os.environ.items() + if key.startswith(prefix) + } + if params["display_name"] in available_accounts: + params.pop("mnemonic") + + account.login(**params) + + now = datetime.datetime.now() + # Node will only return known / fetched messages for this channel. + # Without enabling community archives feature the node can only fetch last 30 days (from store nodes). + to_midnight = lambda date: date.replace(minute=0, second=0, hour=0, microsecond=0) + start_timestamp: datetime.datetime = to_midnight(now - datetime.timedelta(days=30)) + end_timestamp: datetime.datetime = to_midnight(now - datetime.timedelta(days=1)) + + get_file_name = lambda: str(datetime.datetime.now().timestamp()).replace(".", "") + ".pkl" + + for community in account.communities: + account.logger.info(f"Extracting data for {community['name']} from {start_timestamp} to {end_timestamp}") + community["extracted_timestamp"] = datetime.datetime.now() + file_path = os.path.join(community_info_folder, get_file_name()) + with open(file_path, "wb") as f: + pickle.dump(community, f) + + data = account.call_rpc("messaging", "communities") + for result in data["result"]: + if result["id"] != community["id"]: + continue + + extract_timestamp = datetime.datetime.now() + file_path = os.path.join(members_info_folder, "") + members = pd.DataFrame([ + { + "community_id": community["id"], + "member_id": member_id, + "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None, + "extract_timestamp": extract_timestamp + } + for member_id, info in result["members"].items() + ]) + members.to_pickle(os.path.join(members_info_folder, get_file_name())) + break + + for channel in community["channels"]: + messages = account.get_messages(channel["chat_id"], start_timestamp, end_timestamp) + file_path = os.path.join(messages_folder, get_file_name()) + messages = pd.DataFrame(messages) + if len(messages) == 0: + account.logger.info(f"No messages found for # {channel['name']}") + continue + + account.logger.info(f"Extracted {len(messages)} message(s) from # {channel['name']}") + messages.assign( + community_id = community["id"], + extracted_timestamp = datetime.datetime.now() + ).to_pickle(file_path) + +def upload(folder: str, config: dict): + """ + Upload Status App `download` file to Postgres. + NOTE: The Postgres schema must already exist + + Parameters: + - `folder` - the folder where the files will be created. Sub folders are automatically created + - `config` - the `load_config` configuration + """ + path = Path(folder) + table_name_mapping: dict[str, str] = config["postgres"]["tables"] + table_schema = config["postgres"]["schema"] + + upload: dict[str, list] = {} + completed = [] + for file_path in path.rglob("*.pkl"): + + table_name = table_name_mapping.get(file_path.parent.name) + if not table_name: + continue + + data = pd.read_pickle(file_path) + if isinstance(data, dict): + data = pd.DataFrame([data]) + + if table_name not in upload: + upload[table_name] = [] + + upload[table_name].append(data) + completed.append(str(file_path)) + + prefix = "POSTGRES_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + connector = Postgres(**params) + for table_name, data in upload.items(): + if len(data) == 0: + continue + + df = pd.concat(data, ignore_index=True) + json_columns = [ + column + for column in df.columns + if isinstance(df[column].dropna().reset_index(drop=True).iloc[0], (dict, list)) + ] + connector.insert(df, table_name, table_schema, json_columns) + + for file_path in completed: + os.remove(file_path) + +def run_factory(folder: str, config: dict, func: Callable[[str, dict], None]): + """ + Run either `upload` or `download` function + """ + logger = Logger() + logger.info(f"Starting {func.__name__}") + while True: + func(folder, config) + seconds = config["sleep"][func.__name__] * 60 + logger.info(f"Function {func.__name__} sleeping for {config['sleep'][func.__name__]} minute(s)") + time.sleep(seconds) + +if __name__ == "__main__": + folder = os.path.dirname(__file__) + config = load_config(os.path.join(folder, "config.yaml")) + upload_folder = os.path.join(os.path.dirname(__file__), "uploads") + + download_process = Process(target=run_factory, args=(upload_folder, config, download)) + upload_process = Process(target=run_factory, args=(upload_folder, config, upload)) + + upload_process.start() + download_process.start() diff --git a/postgres.py b/postgres.py index 1721fe6..6337fbf 100644 --- a/postgres.py +++ b/postgres.py @@ -5,16 +5,19 @@ https://github.com/status-im/ift-data-py/blob/master/ift_data/clients/postgres.p import psycopg2 import pandas as pd -from typing import Optional +from typing import Optional, Union from sqlalchemy import create_engine from sqlalchemy.dialects.postgresql import JSONB class Postgres: - - def __init__(self, username: str, password: str, port: int = 5432, database: str = "data-warehouse", host: str = "data-01.do-ams3.bi.test.status.im"): - + + def __init__(self, username: str, password: str, port: Union[int, str], database: str, host: str): + + if isinstance(port, str): + port = int(port) + self.__params = { - "host": host, + "host": host, "user": username, "password": password, "port": port, @@ -36,6 +39,7 @@ class Postgres: - `schema` - the name of the schema - `json_columns` - when creating the table, `dict` columns will be turned into JSON objects in Postgres """ + self.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}") engine = create_engine(self.__url) data.columns = [column.lower() for column in data.columns] @@ -52,10 +56,40 @@ class Postgres: json_column: JSONB for json_column in json_columns } - + data.to_sql(**params) + def execute(self, query: str): + """ + Execute queries such as INSERT, UPDATE, DELETE etc. + + Parameters: + - `query` - the PostgreSQL query + """ + self.__execute(query) + self.__conn.commit() def close(self): self.__cursor.close() - self.__conn.close() \ No newline at end of file + self.__conn.close() + + def __del__(self): + self.close() + + def __execute(self, query: str): + + failed = False + is_closed = bool(self.__conn.closed) + + if is_closed: + self.__conn: psycopg2.extensions.connection = psycopg2.connect(**self.__params) + self.__cursor: psycopg2.extensions.cursor = self.__conn.cursor() + + try: + self.__cursor.execute(query) + except psycopg2.errors.InFailedSqlTransaction: + self.__conn.rollback() + failed = True + + if failed: + self.__cursor.execute(query) diff --git a/requirements.txt b/requirements.txt index 014501b..3307083 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,5 @@ -pytest~=7.4.0 -requests~=2.32.4 -websocket-client~=1.4.2 -tenacity~=9.0.0 -docker~=7.1.0 -pyright~=1.1.388 -black~=24.10.0 -pre-commit~=3.6.2 -pytest-xdist~=3.6.1 -pytest-rerunfailures==13.0 -web3~=7.9.0 -matplotlib>=3.5.0 -eth-typing~=5.2.1 -faker~=37.6.0 -qrcode pandas pyyaml python-dotenv psycopg2-binary -sqlalchemy \ No newline at end of file +sqlalchemy diff --git a/resources/__init__.py b/resources/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/resources/constants.py b/resources/constants.py deleted file mode 100644 index 03dcb08..0000000 --- a/resources/constants.py +++ /dev/null @@ -1,142 +0,0 @@ -# Main constants file for tests -from dataclasses import dataclass -import os -from typing import Optional, List, Dict, Any -from resources.test_data import mnemonic_12, mnemonic_15, mnemonic_24 - - -@dataclass -class Account: - address: str - private_key: str - password: str - passphrase: str - accounts: Optional[List[Dict[str, Any]]] = None # Optional list of accounts - profile_data: Optional[Dict[str, Any]] = None # Optional profile data - - -user_1 = Account( - address="0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - private_key="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - password="Strong12345", - passphrase="test test test test test test test test test test test junk", -) -user_2 = Account( - address="0x70997970C51812dc3A010C7d01b50e0d17dc79C8", - private_key="0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", - password="Strong12345", - passphrase="test test test test test test test test test test nest junk", -) - -user_mnemonic_12 = Account( - address="0xC43f4Ab94eC965a3EE9815C5Df07383057d261A8", - private_key="", - password="Strong12345", - passphrase="exhibit soldier miracle series edge atom daring alter absorb decide orphan addict", - accounts=mnemonic_12.accounts, - profile_data=mnemonic_12.profile_data, -) - -user_mnemonic_15 = Account( - address="0x685d7ec8e08769ca7020a6b65709887e38e68e6d", - private_key="", - password="Strong12345", - passphrase="category two chapter fame hunt horse huge rotate inner monkey affair champion mixed tail final", - accounts=mnemonic_15.accounts, - profile_data=mnemonic_15.profile_data, -) - -user_mnemonic_24 = Account( - address="0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", - private_key="", - password="Strong12345", - passphrase=( - "border cabbage grape stage return enable bamboo main only voyage glad race patient stool drum sort " - "army abandon elegant grit cinnamon endless rail drink" - ), - accounts=mnemonic_24.accounts, - profile_data=mnemonic_24.profile_data, -) - -new_account_data_1 = { - "address": "0x1234567890abcdef1234567890abcdef12345678", - "key-uid": "", - "wallet": False, - "chat": False, - "type": "generated", - "path": "m/44'/60'/0'/0/0", - "public-key": "0xabcdef", - "name": "account1", - "emoji": "πŸ”‘", - "colorId": "blue", -} - -new_account_data_2 = { - "address": "0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", - "key-uid": "", - "wallet": False, - "chat": False, - "type": "generated", - "path": "m/44'/60'/0'/0/1", - "public-key": "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - "name": "account2", - "emoji": "πŸ”‘", - "colorId": "blue", -} - -user_keycard_1 = { - "keyUID": "5a0dd657-165a-4810-b800-6005452be42f", - "address": "0x1234567890abcdef1234567890abcdef12345678", - "whisperPrivateKey": "example-whisper-private-key", - "whisperPublicKey": "example-whisper-public-key", - "whisperAddress": "example-whisper-address", - "walletPublicKey": "example-wallet-public-key", - "walletAddress": "0xabcdefabcdefabcdefabcdefabcdefabcdefabcd", - "walletRootAddress": "0xrootaddressrootaddressrootaddressrootaddr", - "eip1581Address": "0xeip1581address1234567890abcdef1234567890", - "encryptionPublicKey": "example-encryption-public-key", -} - -keycard_1 = { - "keycard-uid": "kc-0xab1948", - "keycard-name": "TestKeycard-0xab19", - "accounts-addresses": ["0x5e98dbb30871a33f802a710420bf975095c1645c"], - "key-uid": "", -} - -keypair_name = "ImportedKeypairName" - -wallet_account_details_root = { - "name": keypair_name, - "path": "m", - "emoji": "πŸ”‘", - "colorId": "primary", -} - -wallet_account_details_derivation = { - "name": keypair_name, - "path": "m/44'/60'/0'/0/0", - "emoji": "πŸ”‘", - "colorId": "primary", -} - - -PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) -TESTS_DIR = os.path.join(PROJECT_ROOT, "tests-functional") -SIGNALS_DIR = os.path.join(TESTS_DIR, "signals") -FORGE_OUTPUT_DIR = os.path.join(PROJECT_ROOT, "forge_output") -DEPLOYER_ACCOUNT = user_1 -LOG_SIGNALS_TO_FILE = False # used for debugging purposes -USE_IPV6 = os.getenv("USE_IPV6", "No") - -gas_fee_mode_low = 0 -gas_fee_mode_medium = 1 -gas_fee_mode_high = 2 -gas_fee_mode_custom = 3 - -processor_name_transfer = "Transfer" - -ANVIL_NETWORK_ID = 31337 - -STATUS_CONNECTOR_WS_PORT = 8586 -STATUS_MEDIA_SERVER_PORT = 8587 diff --git a/resources/enums.py b/resources/enums.py deleted file mode 100644 index 818704c..0000000 --- a/resources/enums.py +++ /dev/null @@ -1,56 +0,0 @@ -from enum import Enum - - -class MessageContentType(Enum): - UNKNOWN_CONTENT_TYPE = 0 - TEXT_PLAIN = 1 - STICKER = 2 - STATUS = 3 - EMOJI = 4 - TRANSACTION_COMMAND = 5 - SYSTEM_MESSAGE_CONTENT_PRIVATE_GROUP = 6 - IMAGE = 7 - AUDIO = 8 - COMMUNITY = 9 - SYSTEM_MESSAGE_GAP = 10 - CONTACT_REQUEST = 11 - DISCORD_MESSAGE = 12 - IDENTITY_VERIFICATION = 13 - SYSTEM_MESSAGE_PINNED_MESSAGE = 14 - SYSTEM_MESSAGE_MUTUAL_EVENT_SENT = 15 - SYSTEM_MESSAGE_MUTUAL_EVENT_ACCEPTED = 16 - SYSTEM_MESSAGE_MUTUAL_EVENT_REMOVED = 17 - BRIDGE_MESSAGE = 18 - - -class ChatType(Enum): - UNKNOWN_TYPE = 0 - ONE_TO_ONE = 1 - PUBLIC = 2 - PRIVATE_GROUP_CHAT = 3 - PROFILE = 4 # Deprecated - TIMELINE = 5 # Deprecated - COMMUNITY_CHAT = 6 - - -class MuteType(Enum): - MUTE_FOR15_MIN = 1 - MUTE_FOR1_HR = 2 - MUTE_FOR8_HR = 3 - MUTE_FOR1_WEEK = 4 - MUTE_TILL_UNMUTED = 5 - MUTE_TILL1_MIN = 6 - UNMUTED = 7 - MUTE_FOR24_HR = 8 - - -class ChatPreviewFilterType(Enum): - Community = 0 - NonCommunity = 1 - - -class RequestToJoinState(Enum): - RequestToJoinStatePending = 1 - RequestToJoinStateDeclined = 2 - RequestToJoinStateAccepted = 3 - RequestToJoinStateCanceled = 4 diff --git a/resources/test_data/__init__.py b/resources/test_data/__init__.py deleted file mode 100644 index 02b3d28..0000000 --- a/resources/test_data/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Make test_data a proper Python package diff --git a/resources/test_data/mnemonic_12.py b/resources/test_data/mnemonic_12.py deleted file mode 100644 index 3a34cc1..0000000 --- a/resources/test_data/mnemonic_12.py +++ /dev/null @@ -1,58 +0,0 @@ -# Account data for mnemonic with 12 words - -accounts = [ - { - "address": "0xb01a7dbaaacc92581558a4d178289be7471ba0f4", - "public-key": ( - "0x047fd6e4384b764cb7782bf747ad3fb51e8c54c2b3c2be7029c72d85d16b07f4be6a8703cf0a1d97be4c8712ad" "52cd6a519efdc723faae4935bbaba5c320dde02b" - ), - "path": "m/43'/60'/1581'/0'/0", - "prodPreferredChainIds": "1:10:42161:8453", - "operable": "fully", - "position": -1, - }, - { - "address": "0xc43f4ab94ec965a3ee9815c5df07383057d261a8", - "public-key": ( - "0x041d8f6ebfa662c506d3d11cd407373f8ff2eb4c9ddc61a834864150a908172efbaa37f0de7dbeeea51f02272" "74e6ccd78fa5a07de55716094dbba475ac9e9ab44" - ), - "path": "m/44'/60'/0'/0/0", - "name": "Account 1", - "colorId": "primary", - "hidden": False, - "prodPreferredChainIds": "1:10:42161:8453", - "position": 0, - }, -] - -profile_data = { - "address": "0x5f8e02f9f52709b29c82bd893d9af0f83273a6e4", - "currency": "usd", - "networks/current-network": "", - "dapps-address": "0xc43f4ab94ec965a3ee9815c5df07383057d261a8", - "eip1581-address": "0x803102f704324d4a4f2ddb1c47f6ab31339d0f70", - "key-uid": "0x3231d92c94548d14f097173765a50bebe28fbad8f2267c9e08cc4433a6f219a4", - "latest-derived-path": 0, - "link-preview-request-enabled": True, - "messages-from-contacts-only": False, - "mutual-contact-enabled?": False, - "name": "Anchored Open Wirehair", - "networks/networks": [], - "photo-path": "", - "preview-privacy?": False, - "public-key": ( - "0x047fd6e4384b764cb7782bf747ad3fb51e8c54c2b3c2be7029c72d85d16b07f4be6a8703cf0a1d97be4c8712ad52cd" "6a519efdc723faae4935bbaba5c320dde02b" - ), - "default-sync-period": 777600, - "appearance": 0, - "profile-pictures-show-to": 2, - "profile-pictures-visibility": 2, - "use-mailservers?": True, - "wallet-root-address": "0x1846a7930d0ab03e5a120ebdd46eff3fe0365824", - "send-status-updates?": True, - "show-community-asset-when-sending-tokens?": True, - "display-assets-below-balance-threshold": 100000000, - "url-unfurling-mode": 1, - "compressedKey": "zQ3shoF8xQNaT44MWQxztUXK6DK9UU63PRgJhn1Zd7oYWXJ5K", - "emojiHash": ["πŸ”’", "🀞🏽", "πŸ–οΈ", "πŸ™‡πŸ½β€β™€οΈ", "πŸ™‹πŸΌβ€β™€οΈ", "πŸ™Œ", "πŸ’Ž", "🎞️", "😊", "🦸🏼", "😀", "πŸ‘΅πŸΏ", "πŸ§‘πŸΏβ€πŸ”§", "🀢🏿"], -} diff --git a/resources/test_data/mnemonic_15.py b/resources/test_data/mnemonic_15.py deleted file mode 100644 index fc18fcd..0000000 --- a/resources/test_data/mnemonic_15.py +++ /dev/null @@ -1,39 +0,0 @@ -# Account data for mnemonic with 15 words - -accounts = [ - { - "address": "0x245b7438961de05444645898f9215e5cf0786891", - "public-key": ( - "0x04c898c7763afd577f10efdd9e5d607caafd6d708e6cad8cee1b6d822d6ab148eb4e76d1c8266c8b73c8ce1d76" "699e072ac5844a9a6934abb565044bf619336302" - ), - "path": "m/43'/60'/1581'/0'/0", - "prodPreferredChainIds": "1:10:42161:8453", - "operable": "fully", - "position": -1, - }, - { - "address": "0x685d7ec8e08769ca7020a6b65709887e38e68e6d", - "public-key": ( - "0x0480acddfad1e73c3e70e8e50f82eb1566e3df125736e9fe9042c4df5022c825afe6234021ad8bbb43e0ab0196" "878c2d9e3c8b5a8f266aca72b0e23d1f84464c72" - ), - "path": "m/44'/60'/0'/0/0", - "name": "Account 1", - "colorId": "primary", - "hidden": False, - "prodPreferredChainIds": "1:10:42161:8453", - "position": 0, - }, -] - -profile_data = { - "address": "0x3644a8cc3860606fdee3b95c8825e17933a91647", - "dapps-address": "0x685d7ec8e08769ca7020a6b65709887e38e68e6d", - "eip1581-address": "0xe98734898ff58ac33a1a9c28f732696ec3e6b580", - "key-uid": "0x944c1ce03f83dd1750acee591745d6ef14da90723af86f97b2df7d7282e8dd97", - "name": "Overcooked Lost Grayreefshark", - "public-key": ( - "0x04c898c7763afd577f10efdd9e5d607caafd6d708e6cad8cee1b6d822d6ab148eb4e76d1c8266c8b73c8ce1d76699e" "072ac5844a9a6934abb565044bf619336302" - ), - "wallet-root-address": "0xe89675c9be641ceeca9f250345dc58528c3de93b", - "emojiHash": ["πŸ„πŸΎ", "πŸ‘’", "πŸ‘¨πŸ½β€πŸŽ“", "πŸ…°οΈ", "πŸ‘©πŸΎβ€πŸ³", "πŸͺ€", "πŸ“©", "πŸ„", "⏳", "πŸ‘ΈπŸ»", "πŸš£πŸΎβ€β™‚οΈ", "πŸ‘©πŸ½β€πŸ€β€πŸ‘©πŸ»", "πŸ“€", "πŸŒ’"], -} diff --git a/resources/test_data/mnemonic_24.py b/resources/test_data/mnemonic_24.py deleted file mode 100644 index fdc7c7d..0000000 --- a/resources/test_data/mnemonic_24.py +++ /dev/null @@ -1,39 +0,0 @@ -# Account data for mnemonic with 24 words - -accounts = [ - { - "address": "0x8a6d1f3b9f158f7274ca4be1a3f0056c86e2ccdb", - "public-key": ( - "0x04c25b359fab7fc8989d6325128b06dd9734b38d207dc2ab652e130a5d59852910fd6414694e1f5ce3a9cdd5c1" "f6bec9a425a57bae10c98ff4337adba3bf8c18bb" - ), - "path": "m/43'/60'/1581'/0'/0", - "prodPreferredChainIds": "1:10:42161:8453", - "operable": "fully", - "position": -1, - }, - { - "address": "0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", - "public-key": ( - "0x04218096ceb5420c9b4cfa9d0187a057099540edff0aa5882b0a16b76fc8f0056d1a01930db4981f8885d00137" "c535740c04eec7ebe8bae7ef9fd98338fba31e04" - ), - "path": "m/44'/60'/0'/0/0", - "name": "Account 1", - "colorId": "primary", - "hidden": False, - "prodPreferredChainIds": "1:10:42161:8453", - "position": 0, - }, -] - -profile_data = { - "address": "0xb47386b0074a9ddfd979540f134915d1df8dc3d0", - "dapps-address": "0xf2d58ae5aa880f7c3f65d769296b1061c61e0955", - "eip1581-address": "0xf203f9c33afd10e2d3888289ad2cad81c4b017c4", - "key-uid": "0xcf119f28496e4123dd6d5a4936c5f595ee1a873b11ead5f275098456eb8777c4", - "name": "Selfassured Pesky Mayfly", - "public-key": ( - "0x04c25b359fab7fc8989d6325128b06dd9734b38d207dc2ab652e130a5d59852910fd6414694e1f5ce3a9cdd5c1f6be" "c9a425a57bae10c98ff4337adba3bf8c18bb" - ), - "wallet-root-address": "0x0410bd5715fdd8ccadede1d3131a9180a96e502c", - "emojiHash": ["πŸ‘¦πŸ»", "πŸ•”", "🧜", "πŸ‘©πŸ½β€πŸ€β€πŸ‘¨πŸΌ", "πŸ‘©πŸΏβ€πŸ”§", "πŸ‰", "πŸ§πŸΏβ€β™‚οΈ", "πŸš£πŸΎβ€β™€οΈ", "πŸ«€", "πŸ„πŸΏ", "🌘", "πŸ€΅πŸΌβ€β™€οΈ", "πŸ„πŸΏβ€β™€οΈ", "🎴"], -} diff --git a/resources/test_data/profile_showcase_utils.py b/resources/test_data/profile_showcase_utils.py deleted file mode 100644 index a7b02a6..0000000 --- a/resources/test_data/profile_showcase_utils.py +++ /dev/null @@ -1,93 +0,0 @@ -def dummy_profile_showcase_preferences(with_collectibles: bool): - preferences = { - "communities": [ - { - "communityId": "0x254254546768764565565", - "showcaseVisibility": 3, - "order": 0, - }, - { - "communityId": "0x865241434343432412343", - "showcaseVisibility": 2, - "order": 0, - }, - ], - "accounts": [ - { - "address": "0x0000000000000000000000000033433445133423", - "showcaseVisibility": 3, - "order": 0, - }, - { - "address": "0x0000000000000000000000000032433445133424", - "showcaseVisibility": 2, - "order": 1, - }, - ], - "verifiedTokens": [ - { - "symbol": "ETH", - "showcaseVisibility": 3, - "order": 1, - }, - { - "symbol": "DAI", - "showcaseVisibility": 1, - "order": 2, - }, - { - "symbol": "SNT", - "showcaseVisibility": 0, - "order": 3, - }, - ], - "unverifiedTokens": [ - { - "contractAddress": "0x454525452023452", - "chainId": 11155111, - "showcaseVisibility": 3, - "order": 0, - }, - { - "contractAddress": "0x12312323323233", - "chainId": 1, - "showcaseVisibility": 2, - "order": 1, - }, - ], - "socialLinks": [ - { - "text": "TwitterID", - "url": "https://twitter.com/ethstatus", - "showcaseVisibility": 3, - "order": 1, - }, - { - "text": "TwitterID", - "url": "https://twitter.com/StatusIMBlog", - "showcaseVisibility": 1, - "order": 2, - }, - { - "text": "GithubID", - "url": "https://github.com/status-im", - "showcaseVisibility": 2, - "order": 3, - }, - ], - } - - if with_collectibles: - preferences["collectibles"] = [ - { - "contractAddress": "0x12378534257568678487683576", - "chainId": 1, - "tokenId": "12321389592999903", - "showcaseVisibility": 3, - "order": 0, - } - ] - else: - preferences["collectibles"] = [] - - return preferences diff --git a/resources/utils.py b/resources/utils.py deleted file mode 100644 index a3fee59..0000000 --- a/resources/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -def assert_response_attributes(actual, expected, keys=None): - """ - Assert that all keys in expected (or in keys) match in actual. - Handles both list-of-dicts and dict. - """ - if isinstance(expected, list): - assert isinstance(actual, list), "Expected a list for actual" - assert len(actual) == len(expected), "Length mismatch" - for idx, exp in enumerate(expected): - act = actual[idx] - check_keys = keys or exp.keys() - for key in check_keys: - assert act[key] == exp[key], f"Mismatch for key '{key}': {act[key]} != {exp[key]}" - elif isinstance(expected, dict): - assert isinstance(actual, dict), "Expected a dict for actual" - check_keys = keys or expected.keys() - for key in check_keys: - assert actual[key] == expected[key], f"Mismatch for key '{key}': {actual.get(key)} != {expected[key]}" - else: - raise TypeError("Expected must be a list or dict") diff --git a/upload.py b/upload.py deleted file mode 100644 index b54f1d4..0000000 --- a/upload.py +++ /dev/null @@ -1,154 +0,0 @@ -import constants, data_utils -import json, datetime, os, time, logging -import pandas as pd -from pathlib import Path, PosixPath -from postgres import Postgres - -def get_community_members(file_path: PosixPath) -> pd.DataFrame: - """ - Get latest Status App login time for all members in the community. - - Parameters: - - `file_path` - the `.pkl` file that has overall member information from `data_utils.get_community_info` - - Output: - - processed data to be uploaded to the database - """ - info: dict = pd.read_pickle(file_path) - if not isinstance(info, dict): - return pd.DataFrame() - - data = pd.DataFrame(info["members"]["info"]).assign( - community_total = info["members"]["total"], - community_id = info["community_id"], - channels = len(info["channels"]) - ) - return data - -def get_community_info(file_path: PosixPath) -> pd.DataFrame: - """ - Get latest Status App community information - - Parameters: - - `file_path` - the `.pkl` file that has overall member information from `data_utils.get_community_info` - - Output: - - Single row for the given file. The data is returned in a DataFrame to make data uploading more robust - """ - info: dict = pd.read_pickle(file_path) - # members data can be found in function get_community_members - if isinstance(info, dict): - info.pop("members") - return pd.DataFrame([info]) - - return pd.DataFrame() - -def get_messages(file_paths: list[str]) -> pd.DataFrame: - """ - Convert all of the JSON messages into a DataFrame - - Parameters: - - `file_paths` - the `Path` of `*.json` files - - Output: - - DataFrame with all of the messages - """ - if not file_paths: - return pd.DataFrame() - - raw_data = [] - for file_path in file_paths: - with open(file_path, "r") as f: - data: dict = json.load(f) - - raw_data.append(pd.DataFrame(data["messages"])) - - data = pd.concat(raw_data, ignore_index=True) - for column in data.columns: - if not column.endswith("timestamp"): - continue - data[column] = pd.to_datetime(data[column], unit="s") - - return data.copy() - -def upload(data: dict[str, pd.DataFrame], connector: Postgres, logger: logging.Logger): - """ - Upload the raw data to Postgres. - - Parameters: - - `data` - the raw concatenated data - - `connector` - initialized Postgres connection - """ - schema = constants.CONFIG["postgres"]["schema"] - for data_key, table_name in constants.CONFIG["postgres"]["tables"].items(): - df = data[data_key] - if len(df) == 0: - logger.info(f"No data to upload to {schema}.{table_name}") - continue - - df = df.assign( - upload_timestamp = datetime.datetime.now() - ) - - json_columns = [ - column - for column in df.columns - if isinstance(df[column].dropna().reset_index(drop=True).iloc[0], (dict, list)) - ] - connector.insert(df, table_name, schema, json_columns) - logger.info(f"Uploaded {len(df)} row(s) to {schema}.{table_name}") - -def run(username: str, password: str, host: str, database: str, port: str, logger: logging.Logger): - - completed = [] - path = Path(constants.UPLOAD_PATH) - connector = Postgres(username, password, port, database, host) - logger.info(f"Initialized Postgres connector!") - # NOTE: the keys are the same as in config.yaml -> postgres.tables - data = { - "members": [], - "community": [], - } - - for file_path in path.rglob("*.pkl"): - data["community"].append(get_community_info(file_path)) - data["members"].append(get_community_members(file_path)) - completed.append(file_path) - logger.info(f"Created community and member data for {file_path}") - - for key, value in data.items(): - if not value: - continue - data[key] = pd.concat(value, ignore_index=True) - - file_paths = list(path.rglob("*.json")) - data["messages"] = get_messages(file_paths) - completed += file_paths - - upload(data, connector, logger) - connector.close() - - if not completed: - return - - for file_path in completed: - os.remove(file_path) - logger.info(f"Deleted {file_path}") - -if __name__ == "__main__": - os.makedirs(constants.UPLOAD_PATH, exist_ok=True) - logger = data_utils.get_logger("upload") - params = { - "username": os.getenv("POSTGRES_USERNAME"), - "password": os.getenv("POSTGRES_PASSWORD"), - "host": os.getenv("POSTGRES_HOST"), - "database": os.getenv("POSTGRES_DATABASE"), - "port": int(os.getenv("POSTGRES_PORT")), - "logger": logger - } - - while True: - run(**params) - seconds = 60 * constants.CONFIG["sleep"]["upload"] - logger.info(f"Sleeping for {constants.CONFIG['sleep']['upload']} minutes") - time.sleep(seconds) diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/config.py b/utils/config.py deleted file mode 100644 index e248e39..0000000 --- a/utils/config.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -from typing import List, Iterator - - -def _calculate_port_range(): - executor_number = int(os.getenv("EXECUTOR_NUMBER", 5)) - base_port = 7000 - range_size = 100 - max_port = 65535 - min_port = 1024 - - start_port = base_port + (executor_number * range_size) - end_port = start_port + 20000 - - # Ensure generated ports are within the valid range - if start_port < min_port or end_port > max_port: - raise ValueError(f"Generated port range ({start_port}-{end_port}) is outside the allowed range ({min_port}-{max_port}).") - - return list(range(start_port, end_port)) - - -class Config: - status_backend_port_range: List[int] = _calculate_port_range() - base_dir: str = "" - - status_backend_urls: Iterator[str] | None = None - password: str = "" # FIXME: remove - docker_project_name: str = "" - docker_image: str = "" - codecov_dir: str = "" - logs_dir: str = "" - benchmark_results_dir: str = "" - logout: bool = False - waku_fleets_config: str = "" - waku_fleet: str = "" - push_fleets_config: str = "" - disable_override_networks: bool = False diff --git a/utils/fake.py b/utils/fake.py deleted file mode 100644 index 4d20b89..0000000 --- a/utils/fake.py +++ /dev/null @@ -1,54 +0,0 @@ -import random - -from faker import Faker - -# Use a single English locale to minimize provider loading -_faker = Faker("en") - - -def community_name() -> str: - ALLOWED_SPECIAL_CHARS = (".", "_", "-", " ") - return _faker.word() + _faker.random_element(ALLOWED_SPECIAL_CHARS) + str(_faker.random_number()) - - -def community_channel_name() -> str: - return _faker.word() - - -def emoji() -> str: - return _faker.emoji() - - -def color() -> str: - return _faker.hex_color() - - -def community_description() -> str: - return _faker.sentence() - - -def profile_name() -> str: - length = random.randint(5, 24) - return _faker.pystr(min_chars=length, max_chars=length) - - -def emoji() -> str: - return _faker.emoji() - - -def account_name() -> str: - return _faker.word() - - -def profile_password(length: int = 8) -> str: - # Letters + digits; no special characters to keep compatibility - return _faker.password(length=length, special_chars=False) - - -def community_channel_identity() -> dict: - return { - "displayName": community_channel_name(), - "emoji": emoji(), - "color": color(), - "description": community_description(), - } diff --git a/utils/image_utils.py b/utils/image_utils.py deleted file mode 100644 index 52c0414..0000000 --- a/utils/image_utils.py +++ /dev/null @@ -1,6 +0,0 @@ -class ImageCropRect: - def __init__(self, ax=0, ay=0, bx=0, by=0): - self.ax = ax - self.ay = ay - self.bx = bx - self.by = by diff --git a/utils/keys.py b/utils/keys.py deleted file mode 100644 index 00d78e1..0000000 --- a/utils/keys.py +++ /dev/null @@ -1,18 +0,0 @@ -from hashlib import shake_256 - - -def compress_public_key(public_key): - if not public_key.startswith("0x"): - public_key = "0x" + public_key - if len(public_key) != 132: - raise ValueError("Invalid public key") - x = public_key[4:68] # Extract X coordinate (first 32 bytes after prefix) - y = public_key[68:132] # Extract Y coordinate (last 32 bytes) - prefix = "03" if int(y, 16) % 2 else "02" # Add prefix 02 for even Y, 03 for odd Y - return "0x" + prefix + x - - -def shake256(msg): - h = shake_256() - h.update(msg) - return "0x" + h.hexdigest(64) diff --git a/utils/retry_utils.py b/utils/retry_utils.py deleted file mode 100644 index e71f581..0000000 --- a/utils/retry_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -from time import sleep - - -# To be used when signals related to RPC requests are not present in the peer ndoe, ex for: requestToJoinCommunity. -def retry_call(func, *args, max_retries=40, retry_interval=0.5, **kwargs): - for attempt in range(max_retries): - try: - response = func(*args, **kwargs) - if response: - return response - except Exception as e: - logging.error(f"Attempt {attempt + 1}/{max_retries}: Unexpected error: {e}") - sleep(retry_interval) - raise Exception(f"Failed to execute {func.__name__} in {max_retries * retry_interval} seconds.") diff --git a/utils/wallet_utils.py b/utils/wallet_utils.py deleted file mode 100644 index c993ef5..0000000 --- a/utils/wallet_utils.py +++ /dev/null @@ -1,152 +0,0 @@ -import json -import logging -import resources.constants as constants -from clients.signals import SignalType, WalletEventType - -from utils.config import Config - - -def get_suggested_routes(rpc_client, **kwargs): - required_params = ["uuid", "sendType", "addrFrom", "addrTo", "amountIn", "tokenID", "gasFeeMode"] - input_params = {} - - for key, new_value in kwargs.items(): - input_params[key] = new_value - - for key in required_params: - if key not in input_params: - logging.info(f"Warning: The key '{key}' does not exist in the input_params parameters and will be ignored.") - - params = [input_params] - - rpc_client.prepare_wait_for_signal("wallet.suggested.routes", 1) - _ = rpc_client.wallet_service.get_suggested_routes_async(params) - - routes_signal = rpc_client.wait_for_signal("wallet.suggested.routes") - routes = routes_signal["event"] - - return routes - - -def build_transactions_from_route(rpc_client, uuid): - if uuid is None or uuid == "": - logging.info(f"Warning: provided '{uuid}' does not exist or is empty") - - _ = rpc_client.wallet_service.build_transactions_from_route(uuid) - - wallet_router_sign_transactions_signal = rpc_client.wait_for_signal("wallet.router.sign-transactions") - wallet_router_sign_transactions = wallet_router_sign_transactions_signal["event"] - - assert "signingDetails" in wallet_router_sign_transactions - assert wallet_router_sign_transactions["signingDetails"]["signOnKeycard"] is False - transaction_hashes = wallet_router_sign_transactions["signingDetails"]["hashes"] - - assert transaction_hashes, "Transaction hashes are empty!" - - return wallet_router_sign_transactions - - -def sign_messages(rpc_client, hashes, address): - tx_signatures = {} - - for hash in hashes: - - response = rpc_client.wallet_service.sign_message(hash, address, Config.password) - - assert response and response.startswith("0x"), f"Invalid transaction signature for hash {hash}: {response}" - - tx_signature = response[2:] - - signature = { - "r": tx_signature[:64], - "s": tx_signature[64:128], - "v": tx_signature[128:], - } - - tx_signatures[hash] = signature - return tx_signatures - - -def check_fees(fee_mode, base_fee, max_priority_fee_per_gas, max_fee_per_gas, suggested_fee_levels): - assert base_fee.startswith("0x") - assert max_priority_fee_per_gas.startswith("0x") - assert max_fee_per_gas.startswith("0x") - - base_fee_int = int(base_fee, 16) - max_priority_fee_per_gas_int = int(max_priority_fee_per_gas, 16) - max_fee_per_gas_int = int(max_fee_per_gas, 16) - - low_max_fee_per_gas = int(suggested_fee_levels["low"], 16) - low_priority_max_fee_per_gas = int(suggested_fee_levels["lowPriority"], 16) - medium_max_fee_per_gas = int(suggested_fee_levels["medium"], 16) - medium_priority_max_fee_per_gas = int(suggested_fee_levels["mediumPriority"], 16) - high_max_fee_per_gas = int(suggested_fee_levels["high"], 16) - high_priority_max_fee_per_gas = int(suggested_fee_levels["highPriority"], 16) - - if fee_mode == constants.gas_fee_mode_low: - assert max_fee_per_gas_int == low_max_fee_per_gas - assert max_priority_fee_per_gas_int == low_priority_max_fee_per_gas - assert base_fee_int + max_priority_fee_per_gas_int <= max_fee_per_gas_int - elif fee_mode == constants.gas_fee_mode_medium: - assert max_fee_per_gas_int == medium_max_fee_per_gas - assert max_priority_fee_per_gas_int == medium_priority_max_fee_per_gas - assert base_fee_int + max_priority_fee_per_gas_int <= max_fee_per_gas_int - elif fee_mode == constants.gas_fee_mode_high: - assert max_fee_per_gas_int == high_max_fee_per_gas - assert max_priority_fee_per_gas_int == high_priority_max_fee_per_gas - assert base_fee_int + max_priority_fee_per_gas_int <= max_fee_per_gas_int - elif fee_mode == constants.gas_fee_mode_custom: - assert base_fee_int + max_priority_fee_per_gas_int == max_fee_per_gas_int - else: - assert False, "Invalid gas fee mode" - - -def check_fees_for_path(path_name, gas_fee_mode, check_approval, route): - for path_tx in route: - if path_tx["ProcessorName"] != path_name: - continue - if check_approval: - assert path_tx["ApprovalRequired"] - check_fees( - gas_fee_mode, - path_tx["ApprovalBaseFee"], - path_tx["ApprovalPriorityFee"], - path_tx["ApprovalMaxFeesPerGas"], - path_tx["SuggestedLevelsForMaxFeesPerGas"], - ) - return - check_fees( - gas_fee_mode, path_tx["TxBaseFee"], path_tx["TxPriorityFee"], path_tx["TxMaxFeesPerGas"], path_tx["SuggestedLevelsForMaxFeesPerGas"] - ) - - -def send_router_transactions_with_signatures(rpc_client, uuid, tx_signatures): - rpc_client.prepare_wait_for_signal( - SignalType.WALLET.value, - 1, - lambda signal: signal["event"]["type"] == WalletEventType.TRANSACTIONS_PENDING_TRANSACTION_STATUS_CHANGED.value, - ) - _ = rpc_client.wallet_service.send_router_transactions_with_signatures(uuid, tx_signatures) - event_response = rpc_client.wait_for_signal(SignalType.WALLET.value)["event"] - tx_status = json.loads(event_response["message"].replace("'", '"')) - - assert tx_status["status"] == "Success" - - return tx_status - - -def send_router_transaction(rpc_client, **kwargs): - routes = get_suggested_routes(rpc_client, **kwargs) - assert "Route" in routes, f"No route found: {routes}" - - build_tx = build_transactions_from_route(rpc_client, kwargs.get("uuid")) - - tx_signatures = sign_messages(rpc_client, build_tx["signingDetails"]["hashes"], kwargs.get("addrFrom")) - - tx_status = send_router_transactions_with_signatures(rpc_client, routes["Uuid"], tx_signatures) - return { - "routes": routes, - "build_tx": build_tx, - "tx_signatures": tx_signatures, - "tx_status": tx_status, - } From bf569a0aa54498a66fd85f2f97b8eb9b15fbbe03 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Fri, 20 Mar 2026 11:50:23 +0200 Subject: [PATCH 2/6] bot: add missing requirements.txt --- bot/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/requirements.txt b/bot/requirements.txt index d1b7668..38ab9d0 100644 --- a/bot/requirements.txt +++ b/bot/requirements.txt @@ -1,2 +1,3 @@ requests websocket-client +websockets From 68dacbfc34ac592c75dfa072346a633f657fd0ee Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Mon, 23 Mar 2026 07:36:49 +0200 Subject: [PATCH 3/6] bot: PR changes Make PR changes based on resolved suggestions --- .dockerignore | 11 +++++ .gitignore | 1 - Dockerfile | 2 - config.yaml | 2 +- docker-compose.yaml | 7 +-- monitor.py | 102 ++++++++++++++++++++++++++++++-------------- 6 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6aae033 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.pyc + +# Status Bot +uploads/ +bot/docs diff --git a/.gitignore b/.gitignore index d7caf06..845728e 100644 --- a/.gitignore +++ b/.gitignore @@ -214,7 +214,6 @@ __marimo__/ /uploads *.DS_Store *.pkl -*.dockerignore # Docker volumes for Status /data-dir *.bkp diff --git a/Dockerfile b/Dockerfile index 73f6782..61e2300 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,6 @@ FROM python:3.12-slim WORKDIR /app COPY . . -# In case if the scripts are ran locally instead of with Docker -RUN rm -rf uploads RUN pip install --no-cache-dir -r requirements.txt \ && if [ -f bot/requirements.txt ]; then pip install --no-cache-dir -r bot/requirements.txt; fi diff --git a/config.yaml b/config.yaml index 5e23f54..8a76602 100644 --- a/config.yaml +++ b/config.yaml @@ -6,7 +6,7 @@ postgres: members: "raw_community_activity" sleep: # Values must be in MINUTES - upload: 15 # 15 min + store: 15 # 15 min download: 1440 # 1 day # Parameters based on Account class bot_params: diff --git a/docker-compose.yaml b/docker-compose.yaml index bdff4b5..fc456b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,9 +1,10 @@ --- services: backend: - build: - context: https://github.com/status-im/status-go.git#develop - platform: linux/amd64 + image: harbor.status.im/bi/status-backend:dev + # build: + # context: https://github.com/status-im/status-go.git#develop + # platform: linux/amd64 container_name: status-backend ports: - 8080:8080 diff --git a/monitor.py b/monitor.py index 533e0fb..60b4357 100644 --- a/monitor.py +++ b/monitor.py @@ -32,6 +32,68 @@ def load_config(file_path: str) -> dict: return config +def extract_community_members(account: Account, community_id: str) -> pd.DataFrame: + """ + Extract the community members. Uses manual RPC call function. + + Parameters: + - `account` - logged in Status Bot account + - `community_id` - community ID as it is in `account.communities` + + Output: + - DataFrame with all of the members + """ + data = account.call_rpc("messaging", "communities") + for result in data["result"]: + if result["id"] != community_id: + continue + + extract_timestamp = datetime.datetime.now() + + members = pd.DataFrame([ + { + "community_id": community_id, + "member_id": member_id, + "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None, + "extract_timestamp": extract_timestamp + } + for member_id, info in result["members"].items() + ]) + return members + + return pd.DataFrame() + +def extract_community_channels(account: Account, community: dict, start_timestamp: datetime.datetime, end_timestamp: datetime.datetime) -> pd.DataFrame: + """ + Extract the community channel messages. + + Parameters: + - `account` - logged in Status Bot account + - `community` - the current community from `account` + - `start_timestamp` - start timestamp for message fetching + - `end_timestamp` - end timestamp for message fetching + + Output: + - DataFrame with all of the community messages for the given start and end timestamps + """ + final = [] + for channel in community["channels"]: + messages = account.get_messages(channel["chat_id"], start_timestamp, end_timestamp) + + messages = pd.DataFrame(messages) + if len(messages) == 0: + account.logger.info(f"No messages found for # {channel['name']}") + continue + + account.logger.info(f"Extracted {len(messages)} message(s) from # {channel['name']}") + messages = messages.assign( + community_id = community["id"], + extracted_timestamp = datetime.datetime.now() + ) + final.append(messages) + + return pd.concat(final, ignore_index=True) if final else pd.DataFrame() + def download(folder: str, config: dict): """ Download Status App messages / info from communities and store them in pickle files. @@ -79,40 +141,16 @@ def download(folder: str, config: dict): with open(file_path, "wb") as f: pickle.dump(community, f) - data = account.call_rpc("messaging", "communities") - for result in data["result"]: - if result["id"] != community["id"]: - continue - - extract_timestamp = datetime.datetime.now() - file_path = os.path.join(members_info_folder, "") - members = pd.DataFrame([ - { - "community_id": community["id"], - "member_id": member_id, - "last_checked": datetime.datetime.fromtimestamp(info["last_update_clock"]) if "last_update_clock" in info else None, - "extract_timestamp": extract_timestamp - } - for member_id, info in result["members"].items() - ]) + members = extract_community_members(account, community["id"]) + if len(members) > 0: members.to_pickle(os.path.join(members_info_folder, get_file_name())) - break - for channel in community["channels"]: - messages = account.get_messages(channel["chat_id"], start_timestamp, end_timestamp) + messages = extract_community_channels(account, community, start_timestamp, end_timestamp) + if len(messages) > 0: file_path = os.path.join(messages_folder, get_file_name()) - messages = pd.DataFrame(messages) - if len(messages) == 0: - account.logger.info(f"No messages found for # {channel['name']}") - continue + messages.to_pickle(file_path) - account.logger.info(f"Extracted {len(messages)} message(s) from # {channel['name']}") - messages.assign( - community_id = community["id"], - extracted_timestamp = datetime.datetime.now() - ).to_pickle(file_path) - -def upload(folder: str, config: dict): +def store(folder: str, config: dict): """ Upload Status App `download` file to Postgres. NOTE: The Postgres schema must already exist @@ -183,7 +221,7 @@ if __name__ == "__main__": upload_folder = os.path.join(os.path.dirname(__file__), "uploads") download_process = Process(target=run_factory, args=(upload_folder, config, download)) - upload_process = Process(target=run_factory, args=(upload_folder, config, upload)) + store_process = Process(target=run_factory, args=(upload_folder, config, store)) - upload_process.start() download_process.start() + store_process.start() From 2ab7056699d46114049af16ae27ea55dc010f984 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Mon, 23 Mar 2026 07:38:01 +0200 Subject: [PATCH 4/6] bot: minor bug fix - `Account` would break when the Python script finishes and does not have anything to logout or close - Wrong address was used as `wallet_address` --- bot/account.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bot/account.py b/bot/account.py index daf9a80..c1d025e 100644 --- a/bot/account.py +++ b/bot/account.py @@ -34,6 +34,7 @@ class Account: # As the docker-compose.yaml folder is at the moment # NOTE: This might change for initial release self.__backup_local_folder = os.path.join(os.path.dirname(os.path.dirname(__file__)), "backups") + os.makedirs(self.__backup_local_folder, exist_ok=True) self.__logger = Logger() self.__timestamp_divisor = 1_000 self.__kd_iterations = 256000 @@ -150,7 +151,7 @@ class Account: "display_name": event["display-name"], "bio": event.get("bio", ""), "password": password, - "wallet_address": event["address"], + "wallet_address": event["dapps-address"], "logged_in_timestamp": datetime.datetime.now() } # Messenger can be activated only when logged in @@ -575,8 +576,15 @@ class Account: Handles automatic logout when calling `del` and after running `python` """ - self.logout() - self.__signal.close(None) + try: + self.logout() + except Exception: + pass + + try: + self.__signal.close(None) + except Exception: + pass def call_rpc(self, prefix: str, method_name: str, params: Optional[Union[list, dict]] = None) -> dict: """ From b4df638f9b029fa057c7711f2b6a266e09178617 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Tue, 24 Mar 2026 18:46:36 +0200 Subject: [PATCH 5/6] bot: PR changes (2) --- config.yaml | 6 ++---- monitor.py | 52 ++++++++++++++++++++++++---------------------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/config.yaml b/config.yaml index 8a76602..8b6dbaf 100644 --- a/config.yaml +++ b/config.yaml @@ -4,10 +4,8 @@ postgres: messages: "raw_messages" community: "raw_community_info" members: "raw_community_activity" -sleep: - # Values must be in MINUTES - store: 15 # 15 min - download: 1440 # 1 day +# Value must be in MINUTES +sleep: 1440 # 1 day # Parameters based on Account class bot_params: # localhost - to run code locally with Status Backend Dockerfile diff --git a/monitor.py b/monitor.py index 60b4357..dcd53f5 100644 --- a/monitor.py +++ b/monitor.py @@ -2,8 +2,6 @@ import datetime, os, pickle, yaml, time import pandas as pd from pathlib import Path from dotenv import load_dotenv -from multiprocessing import Process -from collections.abc import Callable # Manual file imports from bot import Account, Logger from postgres import Postgres @@ -132,23 +130,31 @@ def download(folder: str, config: dict): start_timestamp: datetime.datetime = to_midnight(now - datetime.timedelta(days=30)) end_timestamp: datetime.datetime = to_midnight(now - datetime.timedelta(days=1)) - get_file_name = lambda: str(datetime.datetime.now().timestamp()).replace(".", "") + ".pkl" + get_file_name = lambda: str(to_midnight(datetime.datetime.now()).timestamp()).replace(".", "") + ".pkl" for community in account.communities: account.logger.info(f"Extracting data for {community['name']} from {start_timestamp} to {end_timestamp}") community["extracted_timestamp"] = datetime.datetime.now() file_path = os.path.join(community_info_folder, get_file_name()) - with open(file_path, "wb") as f: - pickle.dump(community, f) - members = extract_community_members(account, community["id"]) - if len(members) > 0: - members.to_pickle(os.path.join(members_info_folder, get_file_name())) + if not os.path.exists(file_path): + with open(file_path, "wb") as f: + pickle.dump(community, f) + account.logger.info(f"Created {file_path}") - messages = extract_community_channels(account, community, start_timestamp, end_timestamp) - if len(messages) > 0: - file_path = os.path.join(messages_folder, get_file_name()) - messages.to_pickle(file_path) + file_path = os.path.join(members_info_folder, get_file_name()) + if not os.path.exists(file_path): + members = extract_community_members(account, community["id"]) + if len(members) > 0: + members.to_pickle(file_path) + account.logger.info(f"Created {file_path}") + + file_path = os.path.join(messages_folder, get_file_name()) + if not os.path.exists(file_path): + messages = extract_community_channels(account, community, start_timestamp, end_timestamp) + if len(messages) > 0: + messages.to_pickle(file_path) + account.logger.info(f"Created {file_path}") def store(folder: str, config: dict): """ @@ -203,25 +209,15 @@ def store(folder: str, config: dict): for file_path in completed: os.remove(file_path) -def run_factory(folder: str, config: dict, func: Callable[[str, dict], None]): - """ - Run either `upload` or `download` function - """ - logger = Logger() - logger.info(f"Starting {func.__name__}") - while True: - func(folder, config) - seconds = config["sleep"][func.__name__] * 60 - logger.info(f"Function {func.__name__} sleeping for {config['sleep'][func.__name__]} minute(s)") - time.sleep(seconds) if __name__ == "__main__": folder = os.path.dirname(__file__) config = load_config(os.path.join(folder, "config.yaml")) upload_folder = os.path.join(os.path.dirname(__file__), "uploads") + logger = Logger() - download_process = Process(target=run_factory, args=(upload_folder, config, download)) - store_process = Process(target=run_factory, args=(upload_folder, config, store)) - - download_process.start() - store_process.start() + while True: + download(upload_folder, config) + store(upload_folder, config) + logger.info(f"Sleeping for {config['sleep']} minute(s)") + time.sleep(config["sleep"]) From 9922348735a2879b323535046d43096523007564 Mon Sep 17 00:00:00 2001 From: Nick Ninov Date: Tue, 24 Mar 2026 21:59:44 +0200 Subject: [PATCH 6/6] bot: incorrect messages --- bot/account.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/account.py b/bot/account.py index c1d025e..0b54226 100644 --- a/bot/account.py +++ b/bot/account.py @@ -410,7 +410,10 @@ class Account: """ Listen for new **RAW** messages continuously. Can be used for real time processing. """ - return self.signal.listen("messages.new") + for message in self.signal.listen("messages.new"): + event: dict = message.get("event", {}) + if "chats" in event or "messages" in event: + yield message def get_messages(self, chat_id: str, start_timestamp: Optional[datetime.datetime] = None, end_timestamp: Optional[datetime.datetime] = None) -> list[dict]: """