diff --git a/source-crypto-market-extractor/Dockerfile b/source-crypto-market-extractor/Dockerfile new file mode 100644 index 0000000..0d1e900 --- /dev/null +++ b/source-crypto-market-extractor/Dockerfile @@ -0,0 +1,8 @@ +FROM airbyte/python-connector-base:1.1.0 + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] diff --git a/source-crypto-market-extractor/README.md b/source-crypto-market-extractor/README.md new file mode 100644 index 0000000..4e45b7c --- /dev/null +++ b/source-crypto-market-extractor/README.md @@ -0,0 +1,69 @@ +# Crypto Market Extractor Source + +This is the repository for the Crypto Market Extractor source connector, written in Python. + +## Usage + +This connector fetch coins value on API. + +The Supported API: +* CoinGecko + +### Configuration + +The connector takes the following input: + +```yaml +coins: + type: array + name: coins + description: List of coin to fetch the price. List available at coingecko api under `/coins/list` + items: + type: string +``` + +### Output + +This connector will return a list of coin for each chain with the following models + +* `name`: Name of the coin. +* `price`: Price in USD of the coin +* `date`: Date of the sync. + +## Local development + +### Prerequisites + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` +``` +source .venv/bin/activate +pip install -r requirements.txt +``` + +### Locally running the connector +``` +python main.py spec +python main.py check --config sample_files/coin_list.json +python main.py discover --config sample_files/coin_list.json +python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json +``` + +### Locally running the connector docker image + +```bash +docker build -t airbyte/crypto-market-extractor:dev . +# Running the spec command against your patched connector +docker run airbyte/crypto-market-extractor:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/crypto-market-extractor:dev spec +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev check --config /sample_files/coin_list.json +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev discover --config /sample_files/coin_list.json +docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev read --config /sample_files/coin_list.json --catalog /sample_files/configured_catalog.json diff --git a/source-crypto-market-extractor/main.py b/source-crypto-market-extractor/main.py new file mode 100644 index 0000000..78e3846 --- /dev/null +++ b/source-crypto-market-extractor/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_crypto_market_extractor.run import run + +if __name__ == "__main__": + run() diff --git a/source-crypto-market-extractor/metadata.yaml b/source-crypto-market-extractor/metadata.yaml new file mode 100644 index 0000000..6fc0bfc --- /dev/null +++ b/source-crypto-market-extractor/metadata.yaml @@ -0,0 +1,25 @@ +data: + allowedHosts: + registries: + oss: + enabled: false + cloud: + enabled: false + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 + connectorSubtype: api + connectorType: source + definitionId: ce91e290-be98-4a34-b43b-28330afdc3c9 + dockerImageTag: 0.0.3 + dockerRepository: harbor.status.im/status-im/airbyte/crypto-market-extractor + githubIssueLabel: source-crypto-market-extractor + icon: crypto-market-extractor.svg + license: MIT + name: Crypto Market Extractor + releaseDate: TODO + supportLevel: community + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/crypto-market-extractor + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/source-crypto-market-extractor/requirements.txt b/source-crypto-market-extractor/requirements.txt new file mode 100644 index 0000000..d6e1198 --- /dev/null +++ b/source-crypto-market-extractor/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/source-crypto-market-extractor/sample_files/coin_list.json b/source-crypto-market-extractor/sample_files/coin_list.json new file mode 100644 index 0000000..a3ead77 --- /dev/null +++ b/source-crypto-market-extractor/sample_files/coin_list.json @@ -0,0 +1,6 @@ +{ + "coins": [ + "bitcoin", + "ethereum" + ] +} diff --git a/source-crypto-market-extractor/sample_files/configured_catalog.json b/source-crypto-market-extractor/sample_files/configured_catalog.json new file mode 100644 index 0000000..ef9466c --- /dev/null +++ b/source-crypto-market-extractor/sample_files/configured_catalog.json @@ -0,0 +1,18 @@ +{ + "streams": [ + { + "stream": { + "name": "coin_price", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": [ + "full_refresh", "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/source-crypto-market-extractor/setup.py b/source-crypto-market-extractor/setup.py new file mode 100644 index 0000000..15867af --- /dev/null +++ b/source-crypto-market-extractor/setup.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_crypto_market_extractor", + description="Source implementation for Crypto Market Extractor.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, + entry_points={ + "console_scripts": [ + "source-crypto-market-extractor=source_crypto_market_extractor.run:run", + ], + }, +) diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/__init__.py b/source-crypto-market-extractor/source_crypto_market_extractor/__init__.py new file mode 100644 index 0000000..245094c --- /dev/null +++ b/source-crypto-market-extractor/source_crypto_market_extractor/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCryptoMarketExtractor + +__all__ = ["SourceCryptoMarketExtractor"] diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/run.py b/source-crypto-market-extractor/source_crypto_market_extractor/run.py new file mode 100644 index 0000000..d188376 --- /dev/null +++ b/source-crypto-market-extractor/source_crypto_market_extractor/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import SourceCryptoMarketExtractor + +def run(): + source = SourceCryptoMarketExtractor() + launch(source, sys.argv[1:]) diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/schemas/coin_price.json b/source-crypto-market-extractor/source_crypto_market_extractor/schemas/coin_price.json new file mode 100644 index 0000000..f9c4721 --- /dev/null +++ b/source-crypto-market-extractor/source_crypto_market_extractor/schemas/coin_price.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": [ + "null", + "string" + ] + }, + "date": { + "type": [ + "null", + "string" + ] + }, + "price": { + "type": [ + "null", + "number" + ] + } + } +} diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/source.py b/source-crypto-market-extractor/source_crypto_market_extractor/source.py new file mode 100644 index 0000000..997c89e --- /dev/null +++ b/source-crypto-market-extractor/source_crypto_market_extractor/source.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from datetime import datetime +import requests +import logging +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +logger = logging.getLogger("airbyte") + +class CoinPrice(HttpStream): + url_base = 'https://api.coingecko.com/api/v3/coins/' + + primary_key = None + + def __init__(self, coins: List['str'], **kwargs): + super().__init__(**kwargs) + self.coins= coins + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for coin in self.coins: + yield { + "coin": coin, + } + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"{stream_slice['coin']}/market_chart?vs_currency=usd&days=1&interval=daily&precision=18" + + def parse_response( + self, + response: requests.Response, + stream_slice: Mapping[str, Any] = None, + **kwargs + ) -> Iterable[Mapping]: + logger.info("Parsing Coin Gecko date for %s", stream_slice['coin']) + market_chart = response.json() + yield { + "coin": stream_slices['coin'], + "date": datetime.today().strftime('%Y%m%d_%H%M'), + # The first value of the prices objects is weird + "price": market_chart['prices'][1][1], + } + + +# Source +class SourceCryptoMarketExtractor(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + return [CoinPrice(coins=config['coins'])] diff --git a/source-crypto-market-extractor/source_crypto_market_extractor/spec.yaml b/source-crypto-market-extractor/source_crypto_market_extractor/spec.yaml new file mode 100644 index 0000000..2c0b52b --- /dev/null +++ b/source-crypto-market-extractor/source_crypto_market_extractor/spec.yaml @@ -0,0 +1,14 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Crypto Market Extractor Spec + type: object + required: + - coins + properties: + coins: + type: array + name: coins + description: List of coin to fetch the price. List available at coingecko api under `/coins/list` + items: + type: string