diff --git a/source-social-network-fetcher/README.md b/source-social-network-fetcher/README.md deleted file mode 100644 index 7ff564a..0000000 --- a/source-social-network-fetcher/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Social Network Fetcher Source - -This is the repository for fetching Social Network information, written in Python. - -## Todos - -* [ ] Implements first version based on original script - * [ ] Fetch the Data users of each count. - * [ ] Fetch Tweets details -* [ ] Improve version: - * limit the data fetching based on input date - - -## Usage - -This connector fetch information from different social network based on their API: - -* Twitter: - * Tweets reaction - * [...] - -### Configuration - -The connector takes the following input: - -```yaml -twitter: - - API-KEY - - Account List -``` - -### Output - -The connector will return the following: - - -## Local development - -### Prerequisites - -#### Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` -``` -source .venv/bin/activate -pip install -r requirements.txt -``` - -### Locally running the connector -``` -python main.py spec -python main.py check --config sample_files/coin_list.json -python main.py discover --config sample_files/coin_list.json -python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json -``` - -### Locally running the connector docker image - -```bash -docker build -t airbyte/social-network-fetcher:dev . -# Running the spec command against your patched connector -docker run airbyte/social-network-fetcher:dev spec -```` - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/social-network-fetcher:dev spec -docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev check --config /sample_files/coin_list.json -docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev discover --config /sample_files/coin_list.json -docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev read --config /sample_files/coin_list.json --catalog /sample_files/configured_catalog.json -``` diff --git a/source-social-network-fetcher/source_social_network_fetcher/__init__.py b/source-social-network-fetcher/source_social_network_fetcher/__init__.py deleted file mode 100644 index 6ff2eb4..0000000 --- a/source-social-network-fetcher/source_social_network_fetcher/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from .source import SourceSocialNetworkFetcher - -__all__ = ["SourceSocialNetworkFetcher"] diff --git a/source-social-network-fetcher/Dockerfile b/source-twitter-fetcher/Dockerfile similarity index 100% rename from source-social-network-fetcher/Dockerfile rename to source-twitter-fetcher/Dockerfile diff --git a/source-twitter-fetcher/README.md b/source-twitter-fetcher/README.md new file mode 100644 index 0000000..c9678b7 --- /dev/null +++ b/source-twitter-fetcher/README.md @@ -0,0 +1,70 @@ +# Twitter Fetcher Source + +This is the repository for fetching Twitter information, written in Python. + +## Todos + +* [ ] Implements first version based on original script + * [ ] Fetch the Data users of each count. + * [ ] Fetch Tweets details +* [ ] Improve version: + * limit the data fetching based on input date + + +## Usage + +This connector fetch information from Twitter based on their API: https://developer.twitter.com/en/docs/twitter-api + +### Configuration + +The connector takes the following input: + +```yaml +twitter: + - API-KEY + - Account List +``` + +### Output + +The connector will return the following: + + +## Local development + +### Prerequisites + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` +``` +source .venv/bin/activate +pip install -r requirements.txt +``` + +### Locally running the connector +``` +python main.py spec +python main.py check --config sample_files/config-example.json +python main.py discover --config sample_files/config-example.json +python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json +``` + +### Locally running the connector docker image + +```bash +docker build -t airbyte/twitter-fetcher:dev . +# Running the spec command against your patched connector +docker run airbyte/twitter-fetcher:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/twitter-fetcher:dev spec +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev check --config /sample_files/config-example.json +docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev discover --config /sample_files/config-example.json +docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev read --config /sample_files/config-example.json --catalog /sample_files/configured_catalog.json +``` diff --git a/source-social-network-fetcher/main.py b/source-twitter-fetcher/main.py similarity index 66% rename from source-social-network-fetcher/main.py rename to source-twitter-fetcher/main.py index 55e6f57..7bfaee1 100644 --- a/source-social-network-fetcher/main.py +++ b/source-twitter-fetcher/main.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_social_network_fetcher.run import run +from source_twitter_fetcher.run import run if __name__ == "__main__": run() diff --git a/source-social-network-fetcher/metadata.yaml b/source-twitter-fetcher/metadata.yaml similarity index 71% rename from source-social-network-fetcher/metadata.yaml rename to source-twitter-fetcher/metadata.yaml index 6de1916..7401e30 100644 --- a/source-social-network-fetcher/metadata.yaml +++ b/source-twitter-fetcher/metadata.yaml @@ -10,16 +10,16 @@ data: connectorSubtype: api connectorType: source definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920 - dockerImageTag: '0.0.3' - dockerRepository: status-im/airbyte/social-network-fetcher - githubIssueLabel: source-social-network-fetcher - icon: social-network-fetcher.svg + dockerImageTag: '0.0.1' + dockerRepository: status-im/airbyte/source-twitter-fetcher + githubIssueLabel: source-twitter-fetcher + icon: twitter-fetcher.svg license: MIT - name: Social Network Fetcher + name: Twitter Fetcher releaseDate: TODO supportLevel: community releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/social-network-fetcher + documentationUrl: https://docs.airbyte.com/integrations/sources/twitter-fetcher tags: - language:python metadataSpecVersion: "1.0" diff --git a/source-social-network-fetcher/requirements.txt b/source-twitter-fetcher/requirements.txt similarity index 100% rename from source-social-network-fetcher/requirements.txt rename to source-twitter-fetcher/requirements.txt diff --git a/source-social-network-fetcher/sample_files/config-example.json b/source-twitter-fetcher/sample_files/config-example.json similarity index 100% rename from source-social-network-fetcher/sample_files/config-example.json rename to source-twitter-fetcher/sample_files/config-example.json diff --git a/source-social-network-fetcher/sample_files/configured_catalog.json b/source-twitter-fetcher/sample_files/configured_catalog.json similarity index 100% rename from source-social-network-fetcher/sample_files/configured_catalog.json rename to source-twitter-fetcher/sample_files/configured_catalog.json diff --git a/source-social-network-fetcher/setup.py b/source-twitter-fetcher/setup.py similarity index 76% rename from source-social-network-fetcher/setup.py rename to source-twitter-fetcher/setup.py index d2800b8..b8c2b5c 100644 --- a/source-social-network-fetcher/setup.py +++ b/source-twitter-fetcher/setup.py @@ -17,8 +17,8 @@ TEST_REQUIREMENTS = [ ] setup( - name="source_social_network_fetcher", - description="Source implementation for Social Network Fetcher.", + name="source_twitter_fetcher", + description="Source implementation for Twitter Fetcher.", author="Status", author_email="devops@status.im", packages=find_packages(), @@ -29,7 +29,7 @@ setup( }, entry_points={ "console_scripts": [ - "source-social-network-fetcher=source_social_network_fetcher.run:run", + "source-twitter-fetcher=source_twitter_fetcher.run:run", ], }, ) diff --git a/source-twitter-fetcher/source_twitter_fetcher/__init__.py b/source-twitter-fetcher/source_twitter_fetcher/__init__.py new file mode 100644 index 0000000..ebda78d --- /dev/null +++ b/source-twitter-fetcher/source_twitter_fetcher/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceTwitterFetcher + +__all__ = ["SourceTwitterFetcher"] diff --git a/source-social-network-fetcher/source_social_network_fetcher/run.py b/source-twitter-fetcher/source_twitter_fetcher/run.py similarity index 64% rename from source-social-network-fetcher/source_social_network_fetcher/run.py rename to source-twitter-fetcher/source_twitter_fetcher/run.py index a752963..345a352 100644 --- a/source-social-network-fetcher/source_social_network_fetcher/run.py +++ b/source-twitter-fetcher/source_twitter_fetcher/run.py @@ -6,8 +6,8 @@ import sys from airbyte_cdk.entrypoint import launch -from .source import SourceSocialNetworkFetcher +from .source import SourceTwitterFetcher def run(): - source = SourceSocialNetworkFetcher() + source = SourceTwitterFetcher() launch(source, sys.argv[1:]) diff --git a/source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_account_data.json b/source-twitter-fetcher/source_twitter_fetcher/schemas/twitter_account_data.json similarity index 100% rename from source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_account_data.json rename to source-twitter-fetcher/source_twitter_fetcher/schemas/twitter_account_data.json diff --git a/source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_tweet.json b/source-twitter-fetcher/source_twitter_fetcher/schemas/twitter_tweet.json similarity index 100% rename from source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_tweet.json rename to source-twitter-fetcher/source_twitter_fetcher/schemas/twitter_tweet.json diff --git a/source-social-network-fetcher/source_social_network_fetcher/source.py b/source-twitter-fetcher/source_twitter_fetcher/source.py similarity index 98% rename from source-social-network-fetcher/source_social_network_fetcher/source.py rename to source-twitter-fetcher/source_twitter_fetcher/source.py index 41cbc21..c615987 100644 --- a/source-social-network-fetcher/source_social_network_fetcher/source.py +++ b/source-twitter-fetcher/source_twitter_fetcher/source.py @@ -7,6 +7,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import logging import requests +import time from datetime import datetime from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -120,7 +121,7 @@ class TwitterTweet(HttpSubStream, TwitterAccountData): time.sleep(2) # Source -class SourceSocialNetworkFetcher(AbstractSource): +class SourceTwitterFetcher(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None diff --git a/source-social-network-fetcher/source_social_network_fetcher/spec.yaml b/source-twitter-fetcher/source_twitter_fetcher/spec.yaml similarity index 59% rename from source-social-network-fetcher/source_social_network_fetcher/spec.yaml rename to source-twitter-fetcher/source_twitter_fetcher/spec.yaml index e6673fb..555f6ca 100644 --- a/source-social-network-fetcher/source_social_network_fetcher/spec.yaml +++ b/source-twitter-fetcher/source_twitter_fetcher/spec.yaml @@ -1,21 +1,23 @@ documentationUrl: https://docsurl.com connectionSpecification: $schema: http://json-schema.org/draft-07/schema# - title: Social Network Fetcher + title: Twitter Fetcher type: object required: - api_key - accounts + - start_time properties: api_key: type: string - description: API Key to authentify to twitter + description: "API Key to authentify to twitter" airbyte_secret: true accounts: type: array - description: List of accounts needing to be extracted + description: "List of accounts needing to be extracted" items: type: string start_time: type: string - description: Start date of fetching data + description: "Start date of fetching data" + format: date-time