source-crypto-market-extractor: init new connector

Signed-off-by: Alexis Pentori <alexis@status.im>

source-crypto-market-extractor/Dockerfile

@@ -0,0 +1,8 @@
+FROM airbyte/python-connector-base:1.1.0
+
+COPY . ./airbyte/integration_code
+RUN pip install ./airbyte/integration_code
+
+# The entrypoint and default env vars are already set in the base image
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

source-crypto-market-extractor/README.md

@@ -0,0 +1,69 @@
+# Crypto Market Extractor Source
+
+This is the repository for the Crypto Market Extractor source connector, written in Python.
+
+## Usage
+
+This connector fetch coins value on API.
+
+The Supported API:
+* CoinGecko
+
+### Configuration
+
+The connector takes the following input:
+
+```yaml
+coins:
+  type: array
+  name: coins
+  description: List of coin to fetch the price. List available at coingecko api under `/coins/list`
+  items:
+    type: string
+```
+
+### Output
+
+This connector will return a list of coin for each chain with the following models
+
+* `name`: Name of the coin.
+* `price`: Price in USD of the coin
+* `date`: Date of the sync.
+
+## Local development
+
+### Prerequisites
+
+#### Activate Virtual Environment and install dependencies
+From this connector directory, create a virtual environment:
+```
+python -m venv .venv
+```
+```
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Locally running the connector
+```
+python main.py spec
+python main.py check --config sample_files/coin_list.json
+python main.py discover --config sample_files/coin_list.json
+python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json
+```
+
+### Locally running the connector docker image
+
+```bash
+docker build -t airbyte/crypto-market-extractor:dev .
+# Running the spec command against your patched connector
+docker run airbyte/crypto-market-extractor:dev spec
+````
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/crypto-market-extractor:dev spec
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev check --config /sample_files/coin_list.json
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev discover --config /sample_files/coin_list.json
+docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/crypto-market-extractor:dev read --config /sample_files/coin_list.json --catalog /sample_files/configured_catalog.json

source-crypto-market-extractor/main.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from source_crypto_market_extractor.run import run
+
+if __name__ == "__main__":
+    run()

source-crypto-market-extractor/metadata.yaml

@@ -0,0 +1,25 @@
+data:
+  allowedHosts:
+  registries:
+    oss:
+      enabled: false
+    cloud:
+      enabled: false
+  connectorBuildOptions:
+   baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27
+  connectorSubtype: api
+  connectorType: source
+  definitionId: ce91e290-be98-4a34-b43b-28330afdc3c9
+  dockerImageTag: 0.0.3
+  dockerRepository: harbor.status.im/status-im/airbyte/crypto-market-extractor
+  githubIssueLabel: source-crypto-market-extractor
+  icon: crypto-market-extractor.svg
+  license: MIT
+  name: Crypto Market Extractor
+  releaseDate: TODO
+  supportLevel: community
+  releaseStage: alpha
+  documentationUrl: https://docs.airbyte.com/integrations/sources/crypto-market-extractor
+  tags:
+    - language:python
+metadataSpecVersion: "1.0"

source-crypto-market-extractor/requirements.txt

@@ -0,0 +1 @@
+-e .

source-crypto-market-extractor/sample_files/coin_list.json

@@ -0,0 +1,6 @@
+{
+  "coins": [
+    "bitcoin",
+    "ethereum"
+  ]
+}

source-crypto-market-extractor/sample_files/configured_catalog.json

@@ -0,0 +1,18 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "coin_price",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "type": "object"
+        },
+        "supported_sync_modes": [
+          "full_refresh",  "incremental"
+        ]
+      },
+      "sync_mode":  "incremental",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

source-crypto-market-extractor/setup.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from setuptools import find_packages, setup
+
+MAIN_REQUIREMENTS = [
+    "airbyte-cdk~=0.2",
+]
+
+TEST_REQUIREMENTS = [
+    "requests-mock~=1.9.3",
+    "pytest~=6.2",
+    "pytest-mock~=3.6.1",
+    "connector-acceptance-test",
+]
+
+setup(
+    name="source_crypto_market_extractor",
+    description="Source implementation for Crypto Market Extractor.",
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    packages=find_packages(),
+    install_requires=MAIN_REQUIREMENTS,
+    package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
+    entry_points={
+        "console_scripts": [
+            "source-crypto-market-extractor=source_crypto_market_extractor.run:run",
+        ],
+    },
+)

source-crypto-market-extractor/source_crypto_market_extractor/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from .source import SourceCryptoMarketExtractor
+
+__all__ = ["SourceCryptoMarketExtractor"]

source-crypto-market-extractor/source_crypto_market_extractor/run.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+
+from airbyte_cdk.entrypoint import launch
+from .source import SourceCryptoMarketExtractor
+
+def run():
+    source = SourceCryptoMarketExtractor()
+    launch(source, sys.argv[1:])

source-crypto-market-extractor/source_crypto_market_extractor/schemas/coin_price.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "date": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "price": {
+      "type": [
+        "null",
+        "number"
+      ]
+    }
+  }
+}

source-crypto-market-extractor/source_crypto_market_extractor/source.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
+from datetime import datetime
+import requests
+import logging
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpStream
+from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+
+logger = logging.getLogger("airbyte")
+
+class CoinPrice(HttpStream):
+    url_base = 'https://api.coingecko.com/api/v3/coins/'
+    
+    primary_key = None
+
+    def __init__(self, coins: List['str'],  **kwargs):
+        super().__init__(**kwargs)
+        self.coins= coins
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        return None
+
+
+
+    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
+        for coin in self.coins:
+            yield {
+                "coin":  coin,
+            }
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"{stream_slice['coin']}/market_chart?vs_currency=usd&days=1&interval=daily&precision=18"
+
+    def parse_response(
+        self,
+        response: requests.Response,
+        stream_slice: Mapping[str, Any] = None,
+        **kwargs
+    ) -> Iterable[Mapping]:
+        logger.info("Parsing Coin Gecko date for %s", stream_slice['coin'])
+        market_chart = response.json()
+        yield {
+            "coin": stream_slices['coin'],
+            "date": datetime.today().strftime('%Y%m%d_%H%M'),
+            # The first value of the prices objects is weird
+            "price": market_chart['prices'][1][1],
+        }
+
+
+# Source
+class SourceCryptoMarketExtractor(AbstractSource):
+    def check_connection(self, logger, config) -> Tuple[bool, any]:
+       return True, None
+
+    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        
+        return [CoinPrice(coins=config['coins'])]

source-crypto-market-extractor/source_crypto_market_extractor/spec.yaml

@@ -0,0 +1,14 @@
+documentationUrl: https://docsurl.com
+connectionSpecification:
+  $schema: http://json-schema.org/draft-07/schema#
+  title: Crypto Market Extractor Spec
+  type: object
+  required:
+    - coins
+  properties:
+    coins:
+      type: array
+      name: coins
+      description: List of coin to fetch the price. List available at coingecko api under `/coins/list`
+      items:
+        type: string