From db21750acdb6ee4cea5b0414b3b8f85a85860e90 Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Tue, 5 Dec 2023 18:22:58 +0100 Subject: [PATCH] wallet-fetcher: Fetching data from bitcoin Signed-off-by: Alexis Pentori --- wallet-fetcher/metadata.yaml | 4 + .../sample_files/configured_catalog.json | 16 +++- wallet-fetcher/sample_files/wallet-2.json | 17 +++- wallet-fetcher/sample_files/wallet.json | 8 +- .../{token.json => bitcoin_token.json} | 0 .../schemas/ethereum_token.json | 54 +++++++++++ .../source_wallet_fetcher/source.py | 95 ++++++++++++------- .../source_wallet_fetcher/spec.yaml | 8 ++ 8 files changed, 166 insertions(+), 36 deletions(-) rename wallet-fetcher/source_wallet_fetcher/schemas/{token.json => bitcoin_token.json} (100%) create mode 100644 wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json diff --git a/wallet-fetcher/metadata.yaml b/wallet-fetcher/metadata.yaml index 75fd457..8ad17a6 100644 --- a/wallet-fetcher/metadata.yaml +++ b/wallet-fetcher/metadata.yaml @@ -24,6 +24,10 @@ data: releaseDate: TODO supportLevel: community releaseStage: alpha + suggestedStreams: + streams: + - bitcoin_token + - ethereum_token documentationUrl: https://docs.airbyte.com/integrations/sources/wallet-fetcher tags: - language:python diff --git a/wallet-fetcher/sample_files/configured_catalog.json b/wallet-fetcher/sample_files/configured_catalog.json index 1c0550c..7dc31a3 100644 --- a/wallet-fetcher/sample_files/configured_catalog.json +++ b/wallet-fetcher/sample_files/configured_catalog.json @@ -2,7 +2,21 @@ "streams": [ { "stream": { - "name": "token", + "name": "ethereum_token", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bitcoin_token", "json_schema": { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object" diff --git a/wallet-fetcher/sample_files/wallet-2.json b/wallet-fetcher/sample_files/wallet-2.json index b436714..034903d 100644 --- a/wallet-fetcher/sample_files/wallet-2.json +++ b/wallet-fetcher/sample_files/wallet-2.json @@ -2,11 +2,24 @@ "wallets": [ { "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8", - "name": "wallet-test-1" + "name": "wallet-test-1", + "blockchain": [ + "ETH" + ] }, { "address": "0xdAC17F958D2ee523a2206206994597C13D831ec7", - "name": "wallet-test-2" + "name": "wallet-test-2", + "blockchain": [ + "ETH" + ] + }, + { + "address": "37GRPEcZneQgAGpmvRdcdVQBdxGN1TNs6t", + "name": "test-wallet-btc", + "blockchain": [ + "BTC" + ] } ] } diff --git a/wallet-fetcher/sample_files/wallet.json b/wallet-fetcher/sample_files/wallet.json index 8291503..6d76432 100644 --- a/wallet-fetcher/sample_files/wallet.json +++ b/wallet-fetcher/sample_files/wallet.json @@ -1,5 +1,11 @@ { "wallets": [ - "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8" + { + "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8", + "name": "test-wallet", + "blochain": [ + "ETH" + ] + } ] } diff --git a/wallet-fetcher/source_wallet_fetcher/schemas/token.json b/wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json similarity index 100% rename from wallet-fetcher/source_wallet_fetcher/schemas/token.json rename to wallet-fetcher/source_wallet_fetcher/schemas/bitcoin_token.json diff --git a/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json b/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json new file mode 100644 index 0000000..6013be6 --- /dev/null +++ b/wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "wallet_name": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "symbol": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "address": { + "type": [ + "null", + "string" + ] + }, + "chain": { + "type": [ + "null", + "string" + ] + }, + "balance": { + "type": [ + "null", + "number" + ] + }, + "decimal": { + "type": [ + "null", + "number" + ] + } + } +} diff --git a/wallet-fetcher/source_wallet_fetcher/source.py b/wallet-fetcher/source_wallet_fetcher/source.py index 79bab88..0d6ad2b 100644 --- a/wallet-fetcher/source_wallet_fetcher/source.py +++ b/wallet-fetcher/source_wallet_fetcher/source.py @@ -16,12 +16,10 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator logger = logging.getLogger("airbyte") -# Basic full refresh stream -class Token(HttpStream): - # TODO: Fill in the url base. Required. - url_base = "https://api.ethplorer.io/getAddressInfo/" - # Set this as a noop. +class BitcoinToken(HttpStream): + url_base = "https://blockchain.info/rawaddr/" + primary_key = None def __init__(self, wallets: List[str], **kwargs): @@ -35,7 +33,47 @@ class Token(HttpStream): "name": wallet['name'] } + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"{stream_slice['address']}" + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, + response: requests.Response, + stream_slice: Mapping[str, Any] = None, + **kwargs + ) -> Iterable[Mapping]: + logger.info("Getting Bitcoin Balance information") + bitcoin_data = response.json() + yield { + "wallet_name": stream_slice['name'], + "name":"BTC", + "symbol":"BTC", + "description": "Bitcoin", + "address":"", + "chain": "bitcoin", + "balance": bitcoin_data['final_balance'], + "decimal":8 + } + +class EthereumToken(HttpStream): + + url_base = "https://api.ethplorer.io/getAddressInfo/" + + primary_key = None + + def __init__(self, wallets: List[str], **kwargs): + super().__init__(**kwargs) + self.wallets = wallets + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for wallet in self.wallets: + yield { + "address": wallet['address'], + "name": wallet['name'] + } def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None @@ -43,11 +81,6 @@ class Token(HttpStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"{stream_slice['address']}?apiKey=freekey" - # def request_params( - # self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - # ) -> MutableMapping[str, Any]: - # return {"wallet_address": self.wallet_address} - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, @@ -57,14 +90,14 @@ class Token(HttpStream): eth_data=response.json()['ETH'] yield { "wallet_name": stream_slice['name'], - "name":"ETH", - "symbol":"ETH", - "description": "Native Ethereum token", - "address":"", - "chain": "Ethereum", - "balance":eth_data['rawBalance'] - , "decimal":18 - } + "name":"ETH", + "symbol":"ETH", + "description": "Native Ethereum token", + "address":"", + "chain": "Ethereum", + "balance":eth_data['rawBalance'], + "decimal":18 + } logging.info("Fetching Tokens balance information") tokens_data=response.json()['tokens'] for t in tokens_data: @@ -72,25 +105,23 @@ class Token(HttpStream): yield extract_token(stream_slice['name'], t) except Exception as e: logger.error('Dropping token not valid %s' % t ) + # Source class SourceWalletFetcher(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: - # TODO add a check for each endpoint return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - TODO: Replace the streams below with your own streams. + + bitcoin_wallets: List = [] + ethereum_wallets: List = [] - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - # TODO remove the authenticator if not required. - tokens: List[Token] = [] + for wallet in config['wallets']: + if 'BTC' in wallet['blockchain']: + bitcoin_wallets.append(wallet) + if 'ETH' in wallet['blockchain']: + ethereum_wallets.append(wallet) - # for wallet in config["wallets"]: - # tokens.append( - # Token( - # wallet_address=wallet['address'], - # ) - # ) - return [Token(wallets=config['wallets'])] + return [ + BitcoinToken(wallets=bitcoin_wallets), + EthereumToken(wallets=ethereum_wallets)] diff --git a/wallet-fetcher/source_wallet_fetcher/spec.yaml b/wallet-fetcher/source_wallet_fetcher/spec.yaml index b9272ca..f1955e2 100644 --- a/wallet-fetcher/source_wallet_fetcher/spec.yaml +++ b/wallet-fetcher/source_wallet_fetcher/spec.yaml @@ -20,3 +20,11 @@ connectionSpecification: name: title: Name type: string + blockchain: + title: blockchain + type: array + items: + type: string + enum: + - BTC + - ETH