crypto-market-extractor: Failsafe API

Signed-off-by: Alexis Pentori <alexis@status.im>

source-crypto-market-extractor/README.md

@@ -47,9 +47,9 @@ pip install -r requirements.txt
 ### Locally running the connector
 ```
 python main.py spec
-python main.py check --config sample_files/coin_list.json
-python main.py discover --config sample_files/coin_list.json
-python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json
+python main.py check --config sample_files/config-example.json
+python main.py discover --config sample_files/config-example.json
+python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json
 ```
 
 ### Locally running the connector docker image

source-crypto-market-extractor/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: ce91e290-be98-4a34-b43b-28330afdc3c9
-  dockerImageTag: 0.0.3
+  dockerImageTag: 0.1.0
   dockerRepository: harbor.status.im/status-im/airbyte/crypto-market-extractor
   githubIssueLabel: source-crypto-market-extractor
   icon: crypto-market-extractor.svg

source-crypto-market-extractor/source_crypto_market_extractor/source.py

@@ -44,13 +44,19 @@ class CoinPrice(HttpStream):
         coin=stream_slice["coin"]
         logger.info("Parsing Coin Gecko data for %s", coin)
         market_chart = response.json()
-        yield {
-            "name": coin,
-            "date": datetime.today().strftime('%Y%m%d_%H%M'),
-            # The first value of the prices objects is weird
-            "price": market_chart['prices'][1][1],
-        }
-
+        logger.info("Response: %s", market_chart)
+        data={ "name": coin, "date": datetime.today().strftime('%Y%m%d_%H%M')}
+        try:
+            if len(market_chart) > 1:
+                data['price'] = market_chart['prices'][1][1]
+            elif len(market_chart) == 1:
+                data['price'] = market_chart['prices'][0][1]
+            else:
+                logger.error("Invalid response from API, %s", market_chart)
+                raise "No correct data return"
+        except err:
+            logger.error('An error happened : %s', err)
+        yield data 
 # Source
 class SourceCryptoMarketExtractor(AbstractSource):
     def check_connection(self, logger, config) -> Tuple[bool, any]: