From 0bd61451e578334935be5b4c81cca4de495d74ad Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Fri, 1 Dec 2023 16:03:26 +0100 Subject: [PATCH] wallet-fetcher: changing input to list of address to fix schema issue Using a list of object as input doesnt work in airbyte due to a schema difference. Temporary fix. Removing unused directory Signed-off-by: Alexis Pentori --- wallet-fetcher/acceptance-test-config.yml | 39 --------- wallet-fetcher/integration_tests/__init__.py | 3 - .../integration_tests/abnormal_state.json | 5 -- .../integration_tests/acceptance.py | 16 ---- .../integration_tests/configured_catalog.json | 22 ----- .../integration_tests/invalid_config.json | 3 - .../integration_tests/sample_config.json | 3 - .../integration_tests/sample_state.json | 5 -- .../sample_files/configured_catalog.json | 22 +---- wallet-fetcher/sample_files/wallet-2.json | 10 +-- wallet-fetcher/sample_files/wallet.json | 5 +- wallet-fetcher/secrets/config.json | 3 - .../source_wallet_fetcher/source.py | 8 +- .../source_wallet_fetcher/spec.yaml | 21 +---- wallet-fetcher/unit_tests/__init__.py | 3 - .../unit_tests/test_incremental_streams.py | 59 ------------- wallet-fetcher/unit_tests/test_source.py | 22 ----- wallet-fetcher/unit_tests/test_streams.py | 83 ------------------- 18 files changed, 10 insertions(+), 322 deletions(-) delete mode 100644 wallet-fetcher/acceptance-test-config.yml delete mode 100644 wallet-fetcher/integration_tests/__init__.py delete mode 100644 wallet-fetcher/integration_tests/abnormal_state.json delete mode 100644 wallet-fetcher/integration_tests/acceptance.py delete mode 100644 wallet-fetcher/integration_tests/configured_catalog.json delete mode 100644 wallet-fetcher/integration_tests/invalid_config.json delete mode 100644 wallet-fetcher/integration_tests/sample_config.json delete mode 100644 wallet-fetcher/integration_tests/sample_state.json delete mode 100644 wallet-fetcher/secrets/config.json delete mode 100644 wallet-fetcher/unit_tests/__init__.py delete mode 100644 wallet-fetcher/unit_tests/test_incremental_streams.py delete mode 100644 wallet-fetcher/unit_tests/test_source.py delete mode 100644 wallet-fetcher/unit_tests/test_streams.py diff --git a/wallet-fetcher/acceptance-test-config.yml b/wallet-fetcher/acceptance-test-config.yml deleted file mode 100644 index 5dcb4f2..0000000 --- a/wallet-fetcher/acceptance-test-config.yml +++ /dev/null @@ -1,39 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-wallet-fetcher:dev -acceptance_tests: - spec: - tests: - - spec_path: "source_wallet_fetcher/spec.yaml" - connection: - tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - tests: - - config_path: "secrets/config.json" - basic_read: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: - bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/wallet-fetcher/integration_tests/__init__.py b/wallet-fetcher/integration_tests/__init__.py deleted file mode 100644 index c941b30..0000000 --- a/wallet-fetcher/integration_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# diff --git a/wallet-fetcher/integration_tests/abnormal_state.json b/wallet-fetcher/integration_tests/abnormal_state.json deleted file mode 100644 index 52b0f2c..0000000 --- a/wallet-fetcher/integration_tests/abnormal_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } -} diff --git a/wallet-fetcher/integration_tests/acceptance.py b/wallet-fetcher/integration_tests/acceptance.py deleted file mode 100644 index 9e64092..0000000 --- a/wallet-fetcher/integration_tests/acceptance.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("connector_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies diff --git a/wallet-fetcher/integration_tests/configured_catalog.json b/wallet-fetcher/integration_tests/configured_catalog.json deleted file mode 100644 index 36f0468..0000000 --- a/wallet-fetcher/integration_tests/configured_catalog.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "customers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/wallet-fetcher/integration_tests/invalid_config.json b/wallet-fetcher/integration_tests/invalid_config.json deleted file mode 100644 index f373299..0000000 --- a/wallet-fetcher/integration_tests/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} diff --git a/wallet-fetcher/integration_tests/sample_config.json b/wallet-fetcher/integration_tests/sample_config.json deleted file mode 100644 index ecc4913..0000000 --- a/wallet-fetcher/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/wallet-fetcher/integration_tests/sample_state.json b/wallet-fetcher/integration_tests/sample_state.json deleted file mode 100644 index 3587e57..0000000 --- a/wallet-fetcher/integration_tests/sample_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} diff --git a/wallet-fetcher/sample_files/configured_catalog.json b/wallet-fetcher/sample_files/configured_catalog.json index 5cde3c5..1c0550c 100644 --- a/wallet-fetcher/sample_files/configured_catalog.json +++ b/wallet-fetcher/sample_files/configured_catalog.json @@ -5,27 +5,7 @@ "name": "token", "json_schema": { "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "wallets": { - "type": "array", - "items": { - "type": "object", - "required": [ - "name", - "address" - ], - "properties": { - "name": { - "type": "string" - }, - "address": { - "type": "string" - } - } - } - } - } + "type": "object" }, "supported_sync_modes": [ "full_refresh" diff --git a/wallet-fetcher/sample_files/wallet-2.json b/wallet-fetcher/sample_files/wallet-2.json index 79755f1..c5d5579 100644 --- a/wallet-fetcher/sample_files/wallet-2.json +++ b/wallet-fetcher/sample_files/wallet-2.json @@ -1,12 +1,6 @@ { "wallets": [ - { - "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8", - "name": "first random wallet" - }, - { - "address": "0xdAC17F958D2ee523a2206206994597C13D831ec7", - "name": "second random wallet" - } + "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8", + "0xdAC17F958D2ee523a2206206994597C13D831ec7" ] } diff --git a/wallet-fetcher/sample_files/wallet.json b/wallet-fetcher/sample_files/wallet.json index 0f7678c..8291503 100644 --- a/wallet-fetcher/sample_files/wallet.json +++ b/wallet-fetcher/sample_files/wallet.json @@ -1,8 +1,5 @@ { "wallets": [ - { - "address": "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8", - "name": "test" - } + "0x23f4569002a5A07f0Ecf688142eEB6bcD883eeF8" ] } diff --git a/wallet-fetcher/secrets/config.json b/wallet-fetcher/secrets/config.json deleted file mode 100644 index f5f8933..0000000 --- a/wallet-fetcher/secrets/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO populate with needed configuration for integration tests or delete this file and any references to it. The schema of this file should match what is in your spec.yaml" -} diff --git a/wallet-fetcher/source_wallet_fetcher/source.py b/wallet-fetcher/source_wallet_fetcher/source.py index c0eb5a2..fce2185 100644 --- a/wallet-fetcher/source_wallet_fetcher/source.py +++ b/wallet-fetcher/source_wallet_fetcher/source.py @@ -24,10 +24,9 @@ class Token(HttpStream): # Set this as a noop. primary_key = None - def __init__(self, wallet_address: str, wallet_name: str, **kwargs): + def __init__(self, wallet_address: str, **kwargs): super().__init__(**kwargs) self.wallet_address = wallet_address - self.wallet_name = wallet_name def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None @@ -57,7 +56,7 @@ class Token(HttpStream): tokens_data=response.json()['tokens'] for t in tokens_data: try: - yield extract_token(self.wallet_name, t) + yield extract_token(self.wallet_address, t) except Exception as e: logger.error('Dropping token not valid %s' % t ) # Source @@ -78,8 +77,7 @@ class SourceWalletFetcher(AbstractSource): for wallet in config["wallets"]: tokens.append( Token( - wallet_address=wallet['address'], - wallet_name=wallet['name'] + wallet_address=wallet, ) ) return tokens diff --git a/wallet-fetcher/source_wallet_fetcher/spec.yaml b/wallet-fetcher/source_wallet_fetcher/spec.yaml index 4894011..0eeca24 100644 --- a/wallet-fetcher/source_wallet_fetcher/spec.yaml +++ b/wallet-fetcher/source_wallet_fetcher/spec.yaml @@ -7,23 +7,8 @@ connectionSpecification: - wallets properties: wallets: + title: Wallets + description: "List of wallet to scan" type: array - description: list of wallet items: - name: - type: string - description: Name of the wallet - examples: - - 'Main Wallet' - address: - type: string - description: Adress of the wallet - pattern: ^[a-zA-W0-9]+$ - examples: - - '0x766c77F7f7edC99acdC9475012756B98037a8F69' - chain: - type: string - description: 'Blockchain to scan. Not working yet' - default: ETH - - + type: string diff --git a/wallet-fetcher/unit_tests/__init__.py b/wallet-fetcher/unit_tests/__init__.py deleted file mode 100644 index c941b30..0000000 --- a/wallet-fetcher/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# diff --git a/wallet-fetcher/unit_tests/test_incremental_streams.py b/wallet-fetcher/unit_tests/test_incremental_streams.py deleted file mode 100644 index b972736..0000000 --- a/wallet-fetcher/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_wallet_fetcher.source import IncrementalWalletFetcherStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalWalletFetcherStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalWalletFetcherStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalWalletFetcherStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalWalletFetcherStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalWalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalWalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalWalletFetcherStream, "cursor_field", "dummy_field") - stream = IncrementalWalletFetcherStream() - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalWalletFetcherStream() - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalWalletFetcherStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/wallet-fetcher/unit_tests/test_source.py b/wallet-fetcher/unit_tests/test_source.py deleted file mode 100644 index 88d92dd..0000000 --- a/wallet-fetcher/unit_tests/test_source.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_wallet_fetcher.source import SourceWalletFetcher - - -def test_check_connection(mocker): - source = SourceWalletFetcher() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceWalletFetcher() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/wallet-fetcher/unit_tests/test_streams.py b/wallet-fetcher/unit_tests/test_streams.py deleted file mode 100644 index b929181..0000000 --- a/wallet-fetcher/unit_tests/test_streams.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_wallet_fetcher.source import WalletFetcherStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(WalletFetcherStream, "path", "v0/example_endpoint") - mocker.patch.object(WalletFetcherStream, "primary_key", "test_primary_key") - mocker.patch.object(WalletFetcherStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = WalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = WalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = WalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = WalletFetcherStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = WalletFetcherStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = WalletFetcherStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = WalletFetcherStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time