From 067652009e153db033c985732ddb73baf6446244 Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Thu, 8 Feb 2024 11:06:03 +0100 Subject: [PATCH] crypto-market-extractor: Failsafe API Signed-off-by: Alexis Pentori --- source-crypto-market-extractor/README.md | 6 +++--- source-crypto-market-extractor/metadata.yaml | 2 +- .../{coin_list.json => config-example.json} | 0 .../source_crypto_market_extractor/source.py | 20 ++++++++++++------- 4 files changed, 17 insertions(+), 11 deletions(-) rename source-crypto-market-extractor/sample_files/{coin_list.json => config-example.json} (100%) diff --git a/source-crypto-market-extractor/README.md b/source-crypto-market-extractor/README.md index 4e45b7c..eaa6f16 100644 --- a/source-crypto-market-extractor/README.md +++ b/source-crypto-market-extractor/README.md @@ -47,9 +47,9 @@ pip install -r requirements.txt ### Locally running the connector ``` python main.py spec -python main.py check --config sample_files/coin_list.json -python main.py discover --config sample_files/coin_list.json -python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json +python main.py check --config sample_files/config-example.json +python main.py discover --config sample_files/config-example.json +python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json ``` ### Locally running the connector docker image diff --git a/source-crypto-market-extractor/metadata.yaml b/source-crypto-market-extractor/metadata.yaml index 6fc0bfc..28331f1 100644 --- a/source-crypto-market-extractor/metadata.yaml +++ b/source-crypto-market-extractor/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: ce91e290-be98-4a34-b43b-28330afdc3c9 - dockerImageTag: 0.0.3 + dockerImageTag: 0.1.0 dockerRepository: harbor.status.im/status-im/airbyte/crypto-market-extractor githubIssueLabel: source-crypto-market-extractor icon: crypto-market-extractor.svg diff --git a/source-crypto-market-extractor/sample_files/coin_list.json b/source-crypto-market-extractor/sample_files/config-example.json similarity index 100% rename from source-crypto-market-extractor/sample_files/coin_list.json rename to source-crypto-market-extractor/sample_files/config-example.json diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/source.py b/source-crypto-market-extractor/source_crypto_market_extractor/source.py index 8914787..79cef37 100644 --- a/source-crypto-market-extractor/source_crypto_market_extractor/source.py +++ b/source-crypto-market-extractor/source_crypto_market_extractor/source.py @@ -44,13 +44,19 @@ class CoinPrice(HttpStream): coin=stream_slice["coin"] logger.info("Parsing Coin Gecko data for %s", coin) market_chart = response.json() - yield { - "name": coin, - "date": datetime.today().strftime('%Y%m%d_%H%M'), - # The first value of the prices objects is weird - "price": market_chart['prices'][1][1], - } - + logger.info("Response: %s", market_chart) + data={ "name": coin, "date": datetime.today().strftime('%Y%m%d_%H%M')} + try: + if len(market_chart) > 1: + data['price'] = market_chart['prices'][1][1] + elif len(market_chart) == 1: + data['price'] = market_chart['prices'][0][1] + else: + logger.error("Invalid response from API, %s", market_chart) + raise "No correct data return" + except err: + logger.error('An error happened : %s', err) + yield data # Source class SourceCryptoMarketExtractor(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: