wallet-fetcher: Fetching data from bitcoin

Signed-off-by: Alexis Pentori <alexis@status.im>

wallet-fetcher/metadata.yaml

@@ -24,6 +24,10 @@ data:
   releaseDate: TODO
   supportLevel: community
   releaseStage: alpha
+  suggestedStreams:
+    streams:
+      - bitcoin_token
+      - ethereum_token
   documentationUrl: https://docs.airbyte.com/integrations/sources/wallet-fetcher
   tags:
     - language:python

wallet-fetcher/sample_files/configured_catalog.json

@@ -2,7 +2,21 @@
   "streams": [
     {
       "stream": {
-        "name": "token",
+        "name": "ethereum_token",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "type": "object"
+        },
+        "supported_sync_modes": [
+          "full_refresh"
+        ]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "bitcoin_token",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-04/schema#",
           "type": "object"

wallet-fetcher/sample_files/wallet-2.json

@@ -2,11 +2,24 @@
   "wallets": [
     {
       "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8",
-      "name": "wallet-test-1"
+      "name": "wallet-test-1",
+      "blockchain": [
+        "ETH"
+      ]
     },
     {
       "address": "0xdAC17F958D2ee523a2206206994597C13D831ec7",
-      "name": "wallet-test-2"
+      "name": "wallet-test-2",
+      "blockchain": [
+        "ETH"
+      ]
+    },
+    {
+      "address": "37GRPEcZneQgAGpmvRdcdVQBdxGN1TNs6t",
+      "name": "test-wallet-btc",
+      "blockchain": [
+        "BTC"
+      ]
     }
   ]
 }

wallet-fetcher/sample_files/wallet.json

@@ -1,5 +1,11 @@
 {
   "wallets": [
-    "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8"
+    {
+      "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8",
+      "name": "test-wallet",
+      "blochain": [
+        "ETH"
+      ]
+    }
   ]
 }

wallet-fetcher/source_wallet_fetcher/schemas/ethereum_token.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "wallet_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "symbol": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "description": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "address": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "chain": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "balance": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "decimal": {
+      "type": [
+        "null",
+        "number"
+      ]
+    }
+  }
+}

wallet-fetcher/source_wallet_fetcher/source.py

@@ -16,12 +16,10 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
 
 logger = logging.getLogger("airbyte")
 
-# Basic full refresh stream
-class Token(HttpStream):
-    # TODO: Fill in the url base. Required.
-    url_base = "https://api.ethplorer.io/getAddressInfo/"
 
-    # Set this as a noop.
+class BitcoinToken(HttpStream):
+    url_base = "https://blockchain.info/rawaddr/"
+
     primary_key = None
 
     def __init__(self, wallets: List[str], **kwargs):
@@ -35,7 +33,47 @@ class Token(HttpStream):
                 "name": wallet['name']
             }
 
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"{stream_slice['address']}"
 
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        return None
+
+    def parse_response(
+        self,
+        response: requests.Response,
+        stream_slice: Mapping[str, Any] = None,
+        **kwargs
+    ) -> Iterable[Mapping]:
+        logger.info("Getting Bitcoin Balance information")
+        bitcoin_data = response.json()
+        yield {
+             "wallet_name": stream_slice['name'],
+             "name":"BTC", 
+             "symbol":"BTC", 
+             "description": "Bitcoin", 
+             "address":"", 
+             "chain": "bitcoin",
+             "balance": bitcoin_data['final_balance'],
+             "decimal":8
+        }
+
+class EthereumToken(HttpStream):
+
+    url_base = "https://api.ethplorer.io/getAddressInfo/"
+
+    primary_key = None
+
+    def __init__(self, wallets: List[str], **kwargs):
+        super().__init__(**kwargs)
+        self.wallets = wallets
+
+    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
+        for wallet in self.wallets:
+            yield {
+                "address": wallet['address'], 
+                "name": wallet['name']
+            }
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         return None
@@ -43,11 +81,6 @@ class Token(HttpStream):
     def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         return f"{stream_slice['address']}?apiKey=freekey"
 
-        #    def request_params(
-        #        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
-        #    ) -> MutableMapping[str, Any]:
-        #        return {"wallet_address": self.wallet_address}
-
     def parse_response(self,
         response: requests.Response,
         stream_slice: Mapping[str, Any] = None,
@@ -57,14 +90,14 @@ class Token(HttpStream):
         eth_data=response.json()['ETH']
         yield {
             "wallet_name": stream_slice['name'],
-               "name":"ETH", 
+            "name":"ETH", 
-               "symbol":"ETH", 
+            "symbol":"ETH", 
-               "description": "Native Ethereum token", 
+            "description": "Native Ethereum token", 
-               "address":"", 
+            "address":"", 
-               "chain": "Ethereum",
+            "chain": "Ethereum",
-               "balance":eth_data['rawBalance']
+            "balance":eth_data['rawBalance'],
-               , "decimal":18
+            "decimal":18
-               }
+        }
         logging.info("Fetching Tokens balance information")
         tokens_data=response.json()['tokens']
         for t in tokens_data:
@@ -72,25 +105,23 @@ class Token(HttpStream):
                 yield extract_token(stream_slice['name'], t)
             except Exception as e:
                 logger.error('Dropping token not valid %s' % t )
+
 # Source
 class SourceWalletFetcher(AbstractSource):
     def check_connection(self, logger, config) -> Tuple[bool, any]:
-        # TODO add a check for each endpoint
         return True, None
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        """
+        
-        TODO: Replace the streams below with your own streams.
+        bitcoin_wallets: List = []
+        ethereum_wallets: List = []
 
-        :param config: A Mapping of the user input configuration as defined in the connector spec.
+        for wallet in config['wallets']:
-        """
+            if 'BTC' in wallet['blockchain']:
-        # TODO remove the authenticator if not required.
+                bitcoin_wallets.append(wallet)
-        tokens: List[Token] = []
+            if 'ETH' in wallet['blockchain']:
+                ethereum_wallets.append(wallet)
 
-        #        for wallet in config["wallets"]:
+        return [
-        #            tokens.append(
+            BitcoinToken(wallets=bitcoin_wallets),
-        #                Token(
+            EthereumToken(wallets=ethereum_wallets)]
-        #                    wallet_address=wallet['address'], 
-        #                )
-        #            )
-        return [Token(wallets=config['wallets'])]

wallet-fetcher/source_wallet_fetcher/spec.yaml

@@ -20,3 +20,11 @@ connectionSpecification:
           name:
             title: Name
             type: string
+          blockchain:
+            title: blockchain
+            type: array
+            items:
+              type: string
+              enum:
+                - BTC
+                - ETH