refactoring: bank_balance becoming basic-api-fetcher
adding stream for fetching additional token data Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
parent
067652009e
commit
38866cf78b
|
@ -1,4 +0,0 @@
|
|||
{
|
||||
"url": "https://config.test.bi.status.im/api/bank",
|
||||
"api_key": "something"
|
||||
}
|
|
@ -1,8 +0,0 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from .source import SourceBankBalanceFetcher
|
||||
|
||||
__all__ = ["SourceBankBalanceFetcher"]
|
|
@ -1,7 +1,7 @@
|
|||
# Bank Balance Fetcher Source
|
||||
|
||||
This is the repository for the Bank Balance Fetcher source connector, written in Python.
|
||||
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/bank-balance-fetcher).
|
||||
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/basic-api-fetcher).
|
||||
|
||||
## Local development
|
||||
|
||||
|
@ -31,12 +31,12 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu
|
|||
should work as you expect.
|
||||
|
||||
#### Create credentials
|
||||
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/bank-balance-fetcher)
|
||||
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bank_balance_fetcher/spec.yaml` file.
|
||||
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/basic-api-fetcher)
|
||||
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_basic_api_fetcher/spec.yaml` file.
|
||||
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
||||
See `integration_tests/sample_config.json` for a sample config file.
|
||||
|
||||
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source bank-balance-fetcher test creds`
|
||||
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source basic-api-fetcher test creds`
|
||||
and place them into `secrets/config.json`.
|
||||
|
||||
### Locally running the connector
|
||||
|
@ -55,9 +55,9 @@ You can follow install instructions [here](https://github.com/airbytehq/airbyte/
|
|||
Then running the following command will build your connector:
|
||||
|
||||
```bash
|
||||
airbyte-ci connectors --name source-bank-balance-fetcher build
|
||||
airbyte-ci connectors --name source-basic-api-fetcher build
|
||||
```
|
||||
Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-bank-balance-fetcher:dev`.
|
||||
Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-basic-api-fetcher:dev`.
|
||||
|
||||
##### Customizing our build process
|
||||
When contributing on our connector you might need to customize the build process to add a system dependency or set an env var.
|
||||
|
@ -94,7 +94,7 @@ If you would like to patch our connector and build your own a simple approach wo
|
|||
|
||||
1. Create your own Dockerfile based on the latest version of the connector image.
|
||||
```Dockerfile
|
||||
FROM airbyte/source-bank-balance-fetcher:latest
|
||||
FROM airbyte/source-basic-api-fetcher:latest
|
||||
|
||||
COPY . ./airbyte/integration_code
|
||||
RUN pip install ./airbyte/integration_code
|
||||
|
@ -107,18 +107,18 @@ Please use this as an example. This is not optimized.
|
|||
|
||||
2. Build your image:
|
||||
```bash
|
||||
docker build -t airbyte/source-bank-balance-fetcher:dev .
|
||||
docker build -t airbyte/source-basic-api-fetcher:dev .
|
||||
# Running the spec command against your patched connector
|
||||
docker run airbyte/source-bank-balance-fetcher:dev spec
|
||||
docker run airbyte/source-basic-api-fetcher:dev spec
|
||||
````
|
||||
|
||||
#### Run
|
||||
Then run any of the connector commands as follows:
|
||||
```
|
||||
docker run --rm airbyte/source-bank-balance-fetcher:dev spec
|
||||
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bank-balance-fetcher:dev check --config /secrets/config.json
|
||||
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bank-balance-fetcher:dev discover --config /secrets/config.json
|
||||
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bank-balance-fetcher:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
||||
docker run --rm airbyte/source-basic-api-fetcher:dev spec
|
||||
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-basic-api-fetcher:dev check --config /secrets/config.json
|
||||
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-basic-api-fetcher:dev discover --config /secrets/config.json
|
||||
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-basic-api-fetcher:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
||||
```
|
||||
## Testing
|
||||
Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
|
||||
|
@ -145,7 +145,7 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Connector A
|
|||
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
||||
Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command):
|
||||
```bash
|
||||
airbyte-ci connectors --name source-bank-balance-fetcher test
|
||||
airbyte-ci connectors --name source-basic-api-fetcher test
|
||||
```
|
||||
|
||||
## Dependency Management
|
||||
|
@ -156,10 +156,10 @@ We split dependencies between two groups, dependencies that are:
|
|||
|
||||
### Publishing a new version of the connector
|
||||
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
||||
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bank-balance-fetcher test`
|
||||
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-basic-api-fetcher test`
|
||||
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
||||
3. Make sure the `metadata.yaml` content is up to date.
|
||||
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bank-balance-fetcher.md`).
|
||||
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/basic-api-fetcher.md`).
|
||||
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
||||
6. Pat yourself on the back for being an awesome contributor.
|
||||
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
@ -2,7 +2,7 @@
|
|||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from source_bank_balance_fetcher.run import run
|
||||
from source_basic_api_fetcher.run import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
|
@ -10,16 +10,16 @@ data:
|
|||
connectorSubtype: api
|
||||
connectorType: source
|
||||
definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
|
||||
dockerImageTag: 0.0.2
|
||||
dockerRepository: harbor.status.im/status-im/airbyte/source-bank-balance-fetcher
|
||||
githubIssueLabel: source-bank-balance-fetcher
|
||||
icon: bank-balance-fetcher.svg
|
||||
dockerImageTag: 0.0.1
|
||||
dockerRepository: harbor.status.im/status-im/airbyte/source-basic-api-fetcher
|
||||
githubIssueLabel: source-basic-api-fetcher
|
||||
icon: basic-api-fetcher.svg
|
||||
license: MIT
|
||||
name: Bank Balance Fetcher
|
||||
name: Basic API Fetcher
|
||||
releaseDate: TODO
|
||||
supportLevel: community
|
||||
releaseStage: alpha
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/bank-balance-fetcher
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/basic-api-fetcher
|
||||
tags:
|
||||
- language:python
|
||||
metadataSpecVersion: "1.0"
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"url": "https://config.test.bi.status.im",
|
||||
"api_key": "some-random-key",
|
||||
"endpoints": ["bank", "tokens"]
|
||||
}
|
|
@ -13,6 +13,20 @@
|
|||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "additional_tokens",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
]
|
||||
}
|
|
@ -17,7 +17,7 @@ TEST_REQUIREMENTS = [
|
|||
]
|
||||
|
||||
setup(
|
||||
name="source_bank_balance_fetcher",
|
||||
name="source_basic_api_fetcher",
|
||||
description="Source implementation for Bank Balance.",
|
||||
author="Status",
|
||||
author_email="devops@status.im",
|
||||
|
@ -29,7 +29,7 @@ setup(
|
|||
},
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"source-bank-balance-fetcher=source_bank_balance_fetcher.run:run",
|
||||
"source-basic-api-fetcher=source_basic_api_fetcher.run:run",
|
||||
],
|
||||
},
|
||||
)
|
|
@ -0,0 +1,8 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from .source import SourceBasicApiFetcher
|
||||
|
||||
__all__ = ["SourceBasicApiFetcher"]
|
|
@ -6,8 +6,8 @@
|
|||
import sys
|
||||
|
||||
from airbyte_cdk.entrypoint import launch
|
||||
from .source import SourceBankBalanceFetcher
|
||||
from .source import SourceBasicApiFetcher
|
||||
|
||||
def run():
|
||||
source = SourceBankBalanceFetcher()
|
||||
source = SourceBasicApiFetcher()
|
||||
launch(source, sys.argv[1:])
|
|
@ -0,0 +1,54 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"wallet_name": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"symbol": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"description": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"chain": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"balance": {
|
||||
"type": [
|
||||
"null",
|
||||
"number"
|
||||
]
|
||||
},
|
||||
"decimal": {
|
||||
"type": [
|
||||
"null",
|
||||
"number"
|
||||
]
|
||||
},
|
||||
"tags": {
|
||||
"type": [
|
||||
"null",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -47,7 +47,7 @@
|
|||
"balance_date": {
|
||||
"type": [
|
||||
"null",
|
||||
"number"
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"balance": {
|
|
@ -1,8 +1,6 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from abc import ABC
|
||||
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
|
||||
|
||||
|
@ -13,9 +11,8 @@ from airbyte_cdk.sources.streams.http import HttpStream
|
|||
from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
|
||||
|
||||
# Basic full refresh stream
|
||||
class BankBalance(HttpStream):
|
||||
|
||||
url_base = ""
|
||||
class ApiStream(HttpStream):
|
||||
url_base = ""
|
||||
primary_key = None
|
||||
|
||||
def __init__(self, api_key, url, **kwargs):
|
||||
|
@ -23,11 +20,6 @@ class BankBalance(HttpStream):
|
|||
self.api_key = api_key
|
||||
self.url = url
|
||||
|
||||
def path(
|
||||
self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
return f"{self.url}"
|
||||
|
||||
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
||||
return None
|
||||
|
||||
|
@ -36,17 +28,39 @@ class BankBalance(HttpStream):
|
|||
) -> MutableMapping[str, Any]:
|
||||
return { "API-Key" : f"{self.api_key}"}
|
||||
|
||||
class BankBalance(ApiStream):
|
||||
|
||||
primary_key="account_id"
|
||||
def path(
|
||||
self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
return f"{self.url}/api/bank"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
bank_balances = response.json()
|
||||
for b in bank_balances:
|
||||
yield b
|
||||
data = response.json()
|
||||
for elt in data:
|
||||
yield elt
|
||||
|
||||
class AdditionalTokens(ApiStream):
|
||||
def path(
|
||||
self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
return f"{self.url}/api/tokens"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data = response.json()
|
||||
for elt in data:
|
||||
yield elt
|
||||
|
||||
# Source
|
||||
class SourceBankBalanceFetcher(AbstractSource):
|
||||
class SourceBasicApiFetcher(AbstractSource):
|
||||
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
||||
return True, None
|
||||
|
||||
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
||||
return [BankBalance(config['api_key'], config['url'])]
|
||||
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
||||
streams=[]
|
||||
if 'bank' in config['endpoints']:
|
||||
streams.append(BankBalance(config['api_key'], config['url']))
|
||||
if 'tokens' in config['endpoints']:
|
||||
streams.append(AdditionalTokens(config['api_key'],config['url']))
|
||||
return streams
|
|
@ -6,6 +6,7 @@ connectionSpecification:
|
|||
required:
|
||||
- url
|
||||
- api_key
|
||||
- endpoints
|
||||
properties:
|
||||
url:
|
||||
type: string
|
||||
|
@ -14,3 +15,9 @@ connectionSpecification:
|
|||
type: string
|
||||
description: API Key to use the API
|
||||
airbyte_secret: true
|
||||
endpoints:
|
||||
type: array
|
||||
name: endpoints
|
||||
description: List of endpoints to query in the urls
|
||||
items:
|
||||
type: string
|
Loading…
Reference in New Issue