diff --git a/source-bank-balance-fetcher/sample_files/config-example.json b/source-bank-balance-fetcher/sample_files/config-example.json deleted file mode 100644 index cd0ba44..0000000 --- a/source-bank-balance-fetcher/sample_files/config-example.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "url": "https://config.test.bi.status.im/api/bank", - "api_key": "something" -} diff --git a/source-bank-balance-fetcher/source_bank_balance_fetcher/__init__.py b/source-bank-balance-fetcher/source_bank_balance_fetcher/__init__.py deleted file mode 100644 index d9c9852..0000000 --- a/source-bank-balance-fetcher/source_bank_balance_fetcher/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from .source import SourceBankBalanceFetcher - -__all__ = ["SourceBankBalanceFetcher"] diff --git a/source-bank-balance-fetcher/Dockerfile b/source-basic-api-fetcher/Dockerfile similarity index 100% rename from source-bank-balance-fetcher/Dockerfile rename to source-basic-api-fetcher/Dockerfile diff --git a/source-bank-balance-fetcher/README.md b/source-basic-api-fetcher/README.md similarity index 87% rename from source-bank-balance-fetcher/README.md rename to source-basic-api-fetcher/README.md index eb847cf..77c02f7 100644 --- a/source-bank-balance-fetcher/README.md +++ b/source-basic-api-fetcher/README.md @@ -1,7 +1,7 @@ # Bank Balance Fetcher Source This is the repository for the Bank Balance Fetcher source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/bank-balance-fetcher). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/basic-api-fetcher). ## Local development @@ -31,12 +31,12 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/bank-balance-fetcher) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bank_balance_fetcher/spec.yaml` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/basic-api-fetcher) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_basic_api_fetcher/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source bank-balance-fetcher test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source basic-api-fetcher test creds` and place them into `secrets/config.json`. ### Locally running the connector @@ -55,9 +55,9 @@ You can follow install instructions [here](https://github.com/airbytehq/airbyte/ Then running the following command will build your connector: ```bash -airbyte-ci connectors --name source-bank-balance-fetcher build +airbyte-ci connectors --name source-basic-api-fetcher build ``` -Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-bank-balance-fetcher:dev`. +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-basic-api-fetcher:dev`. ##### Customizing our build process When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. @@ -94,7 +94,7 @@ If you would like to patch our connector and build your own a simple approach wo 1. Create your own Dockerfile based on the latest version of the connector image. ```Dockerfile -FROM airbyte/source-bank-balance-fetcher:latest +FROM airbyte/source-basic-api-fetcher:latest COPY . ./airbyte/integration_code RUN pip install ./airbyte/integration_code @@ -107,18 +107,18 @@ Please use this as an example. This is not optimized. 2. Build your image: ```bash -docker build -t airbyte/source-bank-balance-fetcher:dev . +docker build -t airbyte/source-basic-api-fetcher:dev . # Running the spec command against your patched connector -docker run airbyte/source-bank-balance-fetcher:dev spec +docker run airbyte/source-basic-api-fetcher:dev spec ```` #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-bank-balance-fetcher:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bank-balance-fetcher:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bank-balance-fetcher:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bank-balance-fetcher:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-basic-api-fetcher:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-basic-api-fetcher:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-basic-api-fetcher:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-basic-api-fetcher:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. @@ -145,7 +145,7 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Connector A If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): ```bash -airbyte-ci connectors --name source-bank-balance-fetcher test +airbyte-ci connectors --name source-basic-api-fetcher test ``` ## Dependency Management @@ -156,10 +156,10 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bank-balance-fetcher test` +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-basic-api-fetcher test` 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bank-balance-fetcher.md`). +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/basic-api-fetcher.md`). 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/source-bank-balance-fetcher/main.py b/source-basic-api-fetcher/main.py similarity index 67% rename from source-bank-balance-fetcher/main.py rename to source-basic-api-fetcher/main.py index 7b7a2fb..5c2c09f 100644 --- a/source-bank-balance-fetcher/main.py +++ b/source-basic-api-fetcher/main.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_bank_balance_fetcher.run import run +from source_basic_api_fetcher.run import run if __name__ == "__main__": run() diff --git a/source-bank-balance-fetcher/metadata.yaml b/source-basic-api-fetcher/metadata.yaml similarity index 78% rename from source-bank-balance-fetcher/metadata.yaml rename to source-basic-api-fetcher/metadata.yaml index 610dbf1..bfa2d3a 100644 --- a/source-bank-balance-fetcher/metadata.yaml +++ b/source-basic-api-fetcher/metadata.yaml @@ -10,16 +10,16 @@ data: connectorSubtype: api connectorType: source definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920 - dockerImageTag: 0.0.2 - dockerRepository: harbor.status.im/status-im/airbyte/source-bank-balance-fetcher - githubIssueLabel: source-bank-balance-fetcher - icon: bank-balance-fetcher.svg + dockerImageTag: 0.0.1 + dockerRepository: harbor.status.im/status-im/airbyte/source-basic-api-fetcher + githubIssueLabel: source-basic-api-fetcher + icon: basic-api-fetcher.svg license: MIT - name: Bank Balance Fetcher + name: Basic API Fetcher releaseDate: TODO supportLevel: community releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/bank-balance-fetcher + documentationUrl: https://docs.airbyte.com/integrations/sources/basic-api-fetcher tags: - language:python metadataSpecVersion: "1.0" diff --git a/source-bank-balance-fetcher/requirements.txt b/source-basic-api-fetcher/requirements.txt similarity index 100% rename from source-bank-balance-fetcher/requirements.txt rename to source-basic-api-fetcher/requirements.txt diff --git a/source-basic-api-fetcher/sample_files/config-example.json b/source-basic-api-fetcher/sample_files/config-example.json new file mode 100644 index 0000000..6358dd2 --- /dev/null +++ b/source-basic-api-fetcher/sample_files/config-example.json @@ -0,0 +1,5 @@ +{ + "url": "https://config.test.bi.status.im", + "api_key": "some-random-key", + "endpoints": ["bank", "tokens"] +} diff --git a/source-bank-balance-fetcher/sample_files/configured_catalog.json b/source-basic-api-fetcher/sample_files/configured_catalog.json similarity index 51% rename from source-bank-balance-fetcher/sample_files/configured_catalog.json rename to source-basic-api-fetcher/sample_files/configured_catalog.json index acaadb5..66b4f3e 100644 --- a/source-bank-balance-fetcher/sample_files/configured_catalog.json +++ b/source-basic-api-fetcher/sample_files/configured_catalog.json @@ -13,6 +13,20 @@ }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "additional_tokens", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": [ + "full_refresh", "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" } ] } diff --git a/source-bank-balance-fetcher/setup.py b/source-basic-api-fetcher/setup.py similarity index 85% rename from source-bank-balance-fetcher/setup.py rename to source-basic-api-fetcher/setup.py index ff23c69..bd93470 100644 --- a/source-bank-balance-fetcher/setup.py +++ b/source-basic-api-fetcher/setup.py @@ -17,7 +17,7 @@ TEST_REQUIREMENTS = [ ] setup( - name="source_bank_balance_fetcher", + name="source_basic_api_fetcher", description="Source implementation for Bank Balance.", author="Status", author_email="devops@status.im", @@ -29,7 +29,7 @@ setup( }, entry_points={ "console_scripts": [ - "source-bank-balance-fetcher=source_bank_balance_fetcher.run:run", + "source-basic-api-fetcher=source_basic_api_fetcher.run:run", ], }, ) diff --git a/source-basic-api-fetcher/source_basic_api_fetcher/__init__.py b/source-basic-api-fetcher/source_basic_api_fetcher/__init__.py new file mode 100644 index 0000000..0a199f6 --- /dev/null +++ b/source-basic-api-fetcher/source_basic_api_fetcher/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceBasicApiFetcher + +__all__ = ["SourceBasicApiFetcher"] diff --git a/source-bank-balance-fetcher/source_bank_balance_fetcher/run.py b/source-basic-api-fetcher/source_basic_api_fetcher/run.py similarity index 65% rename from source-bank-balance-fetcher/source_bank_balance_fetcher/run.py rename to source-basic-api-fetcher/source_basic_api_fetcher/run.py index f492b8c..cab7002 100644 --- a/source-bank-balance-fetcher/source_bank_balance_fetcher/run.py +++ b/source-basic-api-fetcher/source_basic_api_fetcher/run.py @@ -6,8 +6,8 @@ import sys from airbyte_cdk.entrypoint import launch -from .source import SourceBankBalanceFetcher +from .source import SourceBasicApiFetcher def run(): - source = SourceBankBalanceFetcher() + source = SourceBasicApiFetcher() launch(source, sys.argv[1:]) diff --git a/source-basic-api-fetcher/source_basic_api_fetcher/schemas/additional_tokens.json b/source-basic-api-fetcher/source_basic_api_fetcher/schemas/additional_tokens.json new file mode 100644 index 0000000..bd4b42e --- /dev/null +++ b/source-basic-api-fetcher/source_basic_api_fetcher/schemas/additional_tokens.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "wallet_name": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "symbol": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "chain": { + "type": [ + "null", + "string" + ] + }, + "balance": { + "type": [ + "null", + "number" + ] + }, + "decimal": { + "type": [ + "null", + "number" + ] + }, + "tags": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/source-bank-balance-fetcher/source_bank_balance_fetcher/schemas/bank_balance.json b/source-basic-api-fetcher/source_basic_api_fetcher/schemas/bank_balance.json similarity index 98% rename from source-bank-balance-fetcher/source_bank_balance_fetcher/schemas/bank_balance.json rename to source-basic-api-fetcher/source_basic_api_fetcher/schemas/bank_balance.json index 77f82eb..c4d3f1b 100644 --- a/source-bank-balance-fetcher/source_bank_balance_fetcher/schemas/bank_balance.json +++ b/source-basic-api-fetcher/source_basic_api_fetcher/schemas/bank_balance.json @@ -47,7 +47,7 @@ "balance_date": { "type": [ "null", - "number" + "string" ] }, "balance": { diff --git a/source-bank-balance-fetcher/source_bank_balance_fetcher/source.py b/source-basic-api-fetcher/source_basic_api_fetcher/source.py similarity index 58% rename from source-bank-balance-fetcher/source_bank_balance_fetcher/source.py rename to source-basic-api-fetcher/source_basic_api_fetcher/source.py index 086a896..cab9dcb 100644 --- a/source-bank-balance-fetcher/source_bank_balance_fetcher/source.py +++ b/source-basic-api-fetcher/source_basic_api_fetcher/source.py @@ -1,8 +1,6 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - - from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -13,9 +11,8 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator # Basic full refresh stream -class BankBalance(HttpStream): - - url_base = "" +class ApiStream(HttpStream): + url_base = "" primary_key = None def __init__(self, api_key, url, **kwargs): @@ -23,11 +20,6 @@ class BankBalance(HttpStream): self.api_key = api_key self.url = url - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return f"{self.url}" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None @@ -36,17 +28,39 @@ class BankBalance(HttpStream): ) -> MutableMapping[str, Any]: return { "API-Key" : f"{self.api_key}"} +class BankBalance(ApiStream): + + primary_key="account_id" + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"{self.url}/api/bank" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - bank_balances = response.json() - for b in bank_balances: - yield b + data = response.json() + for elt in data: + yield elt +class AdditionalTokens(ApiStream): + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"{self.url}/api/tokens" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = response.json() + for elt in data: + yield elt # Source -class SourceBankBalanceFetcher(AbstractSource): +class SourceBasicApiFetcher(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [BankBalance(config['api_key'], config['url'])] + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + streams=[] + if 'bank' in config['endpoints']: + streams.append(BankBalance(config['api_key'], config['url'])) + if 'tokens' in config['endpoints']: + streams.append(AdditionalTokens(config['api_key'],config['url'])) + return streams diff --git a/source-bank-balance-fetcher/source_bank_balance_fetcher/spec.yaml b/source-basic-api-fetcher/source_basic_api_fetcher/spec.yaml similarity index 72% rename from source-bank-balance-fetcher/source_bank_balance_fetcher/spec.yaml rename to source-basic-api-fetcher/source_basic_api_fetcher/spec.yaml index 10eb878..c61d3a5 100644 --- a/source-bank-balance-fetcher/source_bank_balance_fetcher/spec.yaml +++ b/source-basic-api-fetcher/source_basic_api_fetcher/spec.yaml @@ -6,6 +6,7 @@ connectionSpecification: required: - url - api_key + - endpoints properties: url: type: string @@ -14,3 +15,9 @@ connectionSpecification: type: string description: API Key to use the API airbyte_secret: true + endpoints: + type: array + name: endpoints + description: List of endpoints to query in the urls + items: + type: string