From 2bf4aecadead6b8f28078267b4438816c99a04fc Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Tue, 9 Jan 2024 14:46:29 +0100 Subject: [PATCH] wallet-fetcher: adding tags to the wallet and coin information Signed-off-by: Alexis Pentori --- wallet-fetcher/README.md | 177 ++++++------------ wallet-fetcher/metadata.yaml | 2 +- wallet-fetcher/sample_files/wallet-2.json | 9 +- wallet-fetcher/sample_files/wallet.json | 3 +- .../schemas/bitcoin_token.json | 6 + .../schemas/ethereum_token.json | 6 + .../source_wallet_fetcher/source.py | 6 - .../source_wallet_fetcher/spec.yaml | 7 + .../source_wallet_fetcher/stream.py | 10 +- 9 files changed, 89 insertions(+), 137 deletions(-) diff --git a/wallet-fetcher/README.md b/wallet-fetcher/README.md index abddfff..0ced780 100644 --- a/wallet-fetcher/README.md +++ b/wallet-fetcher/README.md @@ -1,43 +1,74 @@ # Wallet Fetcher Source This is the repository for the Wallet Fetcher source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/wallet-fetcher). + +## Usage + +This connector fetch wallet balance on different blockchain. + +The Supported chains: +* Bitcoin +* Ethereum + +### Configuration + +The connector takes the following input: + +```yaml +wallets: + address: + title: Address + type: string + description: Address of the wallet + name: + title: Name + type: string + description: Name of the wallet + blockchain: + title: blockchain + description: List of blockchain concerning the wallets + type: array + items: + type: string + enum: + - BTC + - ETH + tags: + title: tags + type: string + description: List of tags linked to the wallet +``` + + + +### Output + +This connector will return a list of coin for each chain with the following models + +* `wallet_name`: Name of the wallet +* `name`: Name of the coin. +* `symbol`: Symbol of the coin. +* `description`: Description of the ERC-20 Token. +* `address`: Address of the Smart contract for ERC-20 Token. +* `chain`: Name of the blockchain. +* `balance`: Number of token owned. +* `decimal`: Number of decimal for the token. +* `tags`: Tags associated with the wallet owning the token. + ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` #### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv ``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt -pip install '.[tests]' ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/wallet-fetcher) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_wallet_fetcher/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source wallet-fetcher test creds` -and place them into `secrets/config.json`. ### Locally running the connector ``` @@ -49,63 +80,6 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Use `airbyte-ci` to build your connector -The Airbyte way of building this connector is to use our `airbyte-ci` tool. -You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). -Then running the following command will build your connector: - -```bash -airbyte-ci connectors --name source-wallet-fetcher build -``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-wallet-fetcher:dev`. - -##### Customizing our build process -When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. -You can customize our build process by adding a `build_customization.py` module to your connector. -This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. -It will be imported at runtime by our build process and the functions will be called if they exist. - -Here is an example of a `build_customization.py` module: -```python -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Feel free to check the dagger documentation for more information on the Container object and its methods. - # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ - from dagger import Container - - -async def pre_connector_install(base_image_container: Container) -> Container: - return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") - -async def post_connector_install(connector_container: Container) -> Container: - return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") -``` - -#### Build your own connector image -This connector is built using our dynamic built process in `airbyte-ci`. -The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. -The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). -It does not rely on a Dockerfile. - -If you would like to patch our connector and build your own a simple approach would be to: - -1. Create your own Dockerfile based on the latest version of the connector image. -```Dockerfile -FROM airbyte/source-wallet-fetcher:latest - -COPY . ./airbyte/integration_code -RUN pip install ./airbyte/integration_code - -# The entrypoint and default env vars are already set in the base image -# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -``` -Please use this as an example. This is not optimized. - -2. Build your image: ```bash docker build -t airbyte/source-wallet-fetcher:dev . # Running the spec command against your patched connector @@ -120,47 +94,4 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wallet-fetcher:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wallet-fetcher:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-wallet-fetcher:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` - -### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): -```bash -airbyte-ci connectors --name source-wallet-fetcher test -``` - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-wallet-fetcher test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/wallet-fetcher.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/wallet-fetcher/metadata.yaml b/wallet-fetcher/metadata.yaml index 3441441..33d97e2 100644 --- a/wallet-fetcher/metadata.yaml +++ b/wallet-fetcher/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: harbor.status.im/status-im/airbyte/wallet-fetcher githubIssueLabel: source-wallet-fetcher icon: icon.svg diff --git a/wallet-fetcher/sample_files/wallet-2.json b/wallet-fetcher/sample_files/wallet-2.json index 034903d..d601908 100644 --- a/wallet-fetcher/sample_files/wallet-2.json +++ b/wallet-fetcher/sample_files/wallet-2.json @@ -5,21 +5,24 @@ "name": "wallet-test-1", "blockchain": [ "ETH" - ] + ], + "tags": "Multisig" }, { "address": "0xdAC17F958D2ee523a2206206994597C13D831ec7", "name": "wallet-test-2", "blockchain": [ "ETH" - ] + ], + "tags": "DeFi,SmartContract" }, { "address": "37GRPEcZneQgAGpmvRdcdVQBdxGN1TNs6t", "name": "test-wallet-btc", "blockchain": [ "BTC" - ] + ], + "tags": "EOA,Stacking" } ] } diff --git a/wallet-fetcher/sample_files/wallet.json b/wallet-fetcher/sample_files/wallet.json index 6d76432..a8559ad 100644 --- a/wallet-fetcher/sample_files/wallet.json +++ b/wallet-fetcher/sample_files/wallet.json @@ -5,7 +5,8 @@ "name": "test-wallet", "blochain": [ "ETH" - ] + ], + "tags": "Multisig" } ] } diff --git a/wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json b/wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json index 6013be6..d592d9a 100644 --- a/wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json +++ b/wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json @@ -49,6 +49,12 @@ "null", "number" ] + }, + "tags": { + "type": [ + "null", + "string" + ] } } } diff --git a/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json b/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json index 6013be6..d592d9a 100644 --- a/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json +++ b/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json @@ -49,6 +49,12 @@ "null", "number" ] + }, + "tags": { + "type": [ + "null", + "string" + ] } } } diff --git a/wallet-fetcher/source_wallet_fetcher/source.py b/wallet-fetcher/source_wallet_fetcher/source.py index 7ef358c..6fd6f9c 100644 --- a/wallet-fetcher/source_wallet_fetcher/source.py +++ b/wallet-fetcher/source_wallet_fetcher/source.py @@ -1,9 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -#from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from .utils import extract_token from .stream import BitcoinToken, EthereumToken diff --git a/wallet-fetcher/source_wallet_fetcher/spec.yaml b/wallet-fetcher/source_wallet_fetcher/spec.yaml index f1955e2..81b4d1b 100644 --- a/wallet-fetcher/source_wallet_fetcher/spec.yaml +++ b/wallet-fetcher/source_wallet_fetcher/spec.yaml @@ -17,14 +17,21 @@ connectionSpecification: address: title: Address type: string + description: Address of the wallet name: title: Name type: string + description: Name of the wallet blockchain: title: blockchain + description: List of blockchain concerning the wallets type: array items: type: string enum: - BTC - ETH + tags: + title: tags + type: string + description: List of tags linked to the wallet diff --git a/wallet-fetcher/source_wallet_fetcher/stream.py b/wallet-fetcher/source_wallet_fetcher/stream.py index 522573e..87e8797 100644 --- a/wallet-fetcher/source_wallet_fetcher/stream.py +++ b/wallet-fetcher/source_wallet_fetcher/stream.py @@ -21,7 +21,8 @@ class BlockchainStream(HttpStream): for wallet in self.wallets: yield { "address": wallet['address'], - "name": wallet['name'] + "name": wallet['name'], + "tags": wallet['tags'] } def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -53,7 +54,8 @@ class BitcoinToken(BlockchainStream): "address":"", "chain": "bitcoin", "balance": bitcoin_data['final_balance'], - "decimal":8 + "decimal":8, + "tags": stream_slice['tags'] } class EthereumToken(BlockchainStream): @@ -78,7 +80,9 @@ class EthereumToken(BlockchainStream): "address":"", "chain": "Ethereum", "balance":eth_data['rawBalance'], - "decimal":18 + "decimal":18, + "tags": stream_slice['tags'] + } logging.info("Fetching Tokens balance information") tokens_data=response.json()['tokens']