mirror of
https://github.com/status-im/airbyte-custom-connector.git
synced 2025-02-17 03:16:36 +00:00
wallet-fetcher: changing input to list of address to fix schema issue
Using a list of object as input doesnt work in airbyte due to a schema difference. Temporary fix. Removing unused directory Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
Alexis Pentori
parent
1fff0adb42
commit
0bd61451e5
@ -1,39 +0,0 @@
|
||||
# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference)
|
||||
# for more information about how to configure these tests
|
||||
connector_image: airbyte/source-wallet-fetcher:dev
|
||||
acceptance_tests:
|
||||
spec:
|
||||
tests:
|
||||
- spec_path: "source_wallet_fetcher/spec.yaml"
|
||||
connection:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
status: "succeed"
|
||||
- config_path: "integration_tests/invalid_config.json"
|
||||
status: "failed"
|
||||
discovery:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
basic_read:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
empty_streams: []
|
||||
# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
|
||||
# expect_records:
|
||||
# path: "integration_tests/expected_records.jsonl"
|
||||
# extra_fields: no
|
||||
# exact_order: no
|
||||
# extra_records: yes
|
||||
incremental:
|
||||
bypass_reason: "This connector does not implement incremental sync"
|
||||
# TODO uncomment this block this block if your connector implements incremental sync:
|
||||
# tests:
|
||||
# - config_path: "secrets/config.json"
|
||||
# configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
# future_state:
|
||||
# future_state_path: "integration_tests/abnormal_state.json"
|
||||
full_refresh:
|
||||
tests:
|
||||
- config_path: "secrets/config.json"
|
||||
configured_catalog_path: "integration_tests/configured_catalog.json"
|
||||
@ -1,3 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
@ -1,5 +0,0 @@
|
||||
{
|
||||
"todo-stream-name": {
|
||||
"todo-field-name": "todo-abnormal-value"
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
pytest_plugins = ("connector_acceptance_test.plugin",)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def connector_setup():
|
||||
"""This fixture is a placeholder for external resources that acceptance test might require."""
|
||||
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
|
||||
yield
|
||||
# TODO: clean up test dependencies
|
||||
@ -1,22 +0,0 @@
|
||||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "customers",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh"]
|
||||
},
|
||||
"sync_mode": "full_refresh",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "employees",
|
||||
"json_schema": {},
|
||||
"supported_sync_modes": ["full_refresh", "incremental"]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "append"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"todo-wrong-field": "this should be an incomplete config file, used in standard tests"
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"fix-me": "TODO"
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
{
|
||||
"todo-stream-name": {
|
||||
"todo-field-name": "value"
|
||||
}
|
||||
}
|
||||
@ -5,27 +5,7 @@
|
||||
"name": "token",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"wallets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"address"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"address": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh"
|
||||
|
||||
@ -1,12 +1,6 @@
|
||||
{
|
||||
"wallets": [
|
||||
{
|
||||
"address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8",
|
||||
"name": "first random wallet"
|
||||
},
|
||||
{
|
||||
"address": "0xdAC17F958D2ee523a2206206994597C13D831ec7",
|
||||
"name": "second random wallet"
|
||||
}
|
||||
"0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8",
|
||||
"0xdAC17F958D2ee523a2206206994597C13D831ec7"
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
{
|
||||
"wallets": [
|
||||
{
|
||||
"address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8",
|
||||
"name": "test"
|
||||
}
|
||||
"0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8"
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"fix-me": "TODO populate with needed configuration for integration tests or delete this file and any references to it. The schema of this file should match what is in your spec.yaml"
|
||||
}
|
||||
@ -24,10 +24,9 @@ class Token(HttpStream):
|
||||
# Set this as a noop.
|
||||
primary_key = None
|
||||
|
||||
def __init__(self, wallet_address: str, wallet_name: str, **kwargs):
|
||||
def __init__(self, wallet_address: str, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.wallet_address = wallet_address
|
||||
self.wallet_name = wallet_name
|
||||
|
||||
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
||||
return None
|
||||
@ -57,7 +56,7 @@ class Token(HttpStream):
|
||||
tokens_data=response.json()['tokens']
|
||||
for t in tokens_data:
|
||||
try:
|
||||
yield extract_token(self.wallet_name, t)
|
||||
yield extract_token(self.wallet_address, t)
|
||||
except Exception as e:
|
||||
logger.error('Dropping token not valid %s' % t )
|
||||
# Source
|
||||
@ -78,8 +77,7 @@ class SourceWalletFetcher(AbstractSource):
|
||||
for wallet in config["wallets"]:
|
||||
tokens.append(
|
||||
Token(
|
||||
wallet_address=wallet['address'],
|
||||
wallet_name=wallet['name']
|
||||
wallet_address=wallet,
|
||||
)
|
||||
)
|
||||
return tokens
|
||||
|
||||
@ -7,23 +7,8 @@ connectionSpecification:
|
||||
- wallets
|
||||
properties:
|
||||
wallets:
|
||||
title: Wallets
|
||||
description: "List of wallet to scan"
|
||||
type: array
|
||||
description: list of wallet
|
||||
items:
|
||||
name:
|
||||
type: string
|
||||
description: Name of the wallet
|
||||
examples:
|
||||
- 'Main Wallet'
|
||||
address:
|
||||
type: string
|
||||
description: Adress of the wallet
|
||||
pattern: ^[a-zA-W0-9]+$
|
||||
examples:
|
||||
- '0x766c77F7f7edC99acdC9475012756B98037a8F69'
|
||||
chain:
|
||||
type: string
|
||||
description: 'Blockchain to scan. Not working yet'
|
||||
default: ETH
|
||||
|
||||
|
||||
type: string
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
@ -1,59 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from airbyte_cdk.models import SyncMode
|
||||
from pytest import fixture
|
||||
from source_wallet_fetcher.source import IncrementalWalletFetcherStream
|
||||
|
||||
|
||||
@fixture
|
||||
def patch_incremental_base_class(mocker):
|
||||
# Mock abstract methods to enable instantiating abstract class
|
||||
mocker.patch.object(IncrementalWalletFetcherStream, "path", "v0/example_endpoint")
|
||||
mocker.patch.object(IncrementalWalletFetcherStream, "primary_key", "test_primary_key")
|
||||
mocker.patch.object(IncrementalWalletFetcherStream, "__abstractmethods__", set())
|
||||
|
||||
|
||||
def test_cursor_field(patch_incremental_base_class):
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
# TODO: replace this with your expected cursor field
|
||||
expected_cursor_field = []
|
||||
assert stream.cursor_field == expected_cursor_field
|
||||
|
||||
|
||||
def test_get_updated_state(patch_incremental_base_class):
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"current_stream_state": None, "latest_record": None}
|
||||
# TODO: replace this with your expected updated stream state
|
||||
expected_state = {}
|
||||
assert stream.get_updated_state(**inputs) == expected_state
|
||||
|
||||
|
||||
def test_stream_slices(patch_incremental_base_class):
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}}
|
||||
# TODO: replace this with your expected stream slices list
|
||||
expected_stream_slice = [None]
|
||||
assert stream.stream_slices(**inputs) == expected_stream_slice
|
||||
|
||||
|
||||
def test_supports_incremental(patch_incremental_base_class, mocker):
|
||||
mocker.patch.object(IncrementalWalletFetcherStream, "cursor_field", "dummy_field")
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
assert stream.supports_incremental
|
||||
|
||||
|
||||
def test_source_defined_cursor(patch_incremental_base_class):
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
assert stream.source_defined_cursor
|
||||
|
||||
|
||||
def test_stream_checkpoint_interval(patch_incremental_base_class):
|
||||
stream = IncrementalWalletFetcherStream()
|
||||
# TODO: replace this with your expected checkpoint interval
|
||||
expected_checkpoint_interval = None
|
||||
assert stream.state_checkpoint_interval == expected_checkpoint_interval
|
||||
@ -1,22 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from source_wallet_fetcher.source import SourceWalletFetcher
|
||||
|
||||
|
||||
def test_check_connection(mocker):
|
||||
source = SourceWalletFetcher()
|
||||
logger_mock, config_mock = MagicMock(), MagicMock()
|
||||
assert source.check_connection(logger_mock, config_mock) == (True, None)
|
||||
|
||||
|
||||
def test_streams(mocker):
|
||||
source = SourceWalletFetcher()
|
||||
config_mock = MagicMock()
|
||||
streams = source.streams(config_mock)
|
||||
# TODO: replace this with your streams number
|
||||
expected_streams_number = 2
|
||||
assert len(streams) == expected_streams_number
|
||||
@ -1,83 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from source_wallet_fetcher.source import WalletFetcherStream
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_base_class(mocker):
|
||||
# Mock abstract methods to enable instantiating abstract class
|
||||
mocker.patch.object(WalletFetcherStream, "path", "v0/example_endpoint")
|
||||
mocker.patch.object(WalletFetcherStream, "primary_key", "test_primary_key")
|
||||
mocker.patch.object(WalletFetcherStream, "__abstractmethods__", set())
|
||||
|
||||
|
||||
def test_request_params(patch_base_class):
|
||||
stream = WalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
|
||||
# TODO: replace this with your expected request parameters
|
||||
expected_params = {}
|
||||
assert stream.request_params(**inputs) == expected_params
|
||||
|
||||
|
||||
def test_next_page_token(patch_base_class):
|
||||
stream = WalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"response": MagicMock()}
|
||||
# TODO: replace this with your expected next page token
|
||||
expected_token = None
|
||||
assert stream.next_page_token(**inputs) == expected_token
|
||||
|
||||
|
||||
def test_parse_response(patch_base_class):
|
||||
stream = WalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"response": MagicMock()}
|
||||
# TODO: replace this with your expected parced object
|
||||
expected_parsed_object = {}
|
||||
assert next(stream.parse_response(**inputs)) == expected_parsed_object
|
||||
|
||||
|
||||
def test_request_headers(patch_base_class):
|
||||
stream = WalletFetcherStream()
|
||||
# TODO: replace this with your input parameters
|
||||
inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
|
||||
# TODO: replace this with your expected request headers
|
||||
expected_headers = {}
|
||||
assert stream.request_headers(**inputs) == expected_headers
|
||||
|
||||
|
||||
def test_http_method(patch_base_class):
|
||||
stream = WalletFetcherStream()
|
||||
# TODO: replace this with your expected http request method
|
||||
expected_method = "GET"
|
||||
assert stream.http_method == expected_method
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("http_status", "should_retry"),
|
||||
[
|
||||
(HTTPStatus.OK, False),
|
||||
(HTTPStatus.BAD_REQUEST, False),
|
||||
(HTTPStatus.TOO_MANY_REQUESTS, True),
|
||||
(HTTPStatus.INTERNAL_SERVER_ERROR, True),
|
||||
],
|
||||
)
|
||||
def test_should_retry(patch_base_class, http_status, should_retry):
|
||||
response_mock = MagicMock()
|
||||
response_mock.status_code = http_status
|
||||
stream = WalletFetcherStream()
|
||||
assert stream.should_retry(response_mock) == should_retry
|
||||
|
||||
|
||||
def test_backoff_time(patch_base_class):
|
||||
response_mock = MagicMock()
|
||||
stream = WalletFetcherStream()
|
||||
expected_backoff_time = None
|
||||
assert stream.backoff_time(response_mock) == expected_backoff_time
|
||||
Loading…
x
Reference in New Issue
Block a user