renaming source-social-network-fetcher into source-twitter-fetcher

Signed-off-by: Alexis Pentori <alexis@status.im>

source-social-network-fetcher/README.md

@@ -1,74 +0,0 @@
-# Social Network Fetcher Source
-
-This is the repository for fetching Social Network information, written in Python.
-
-## Todos
-
-* [ ] Implements first version based on original script
-  * [ ] Fetch the Data users of each count.
-  * [ ] Fetch Tweets details 
-* [ ] Improve version:
-  * limit the data fetching based on input date
-
-
-## Usage
-
-This connector fetch information from different social network based on their API:
-
-* Twitter:
-  * Tweets reaction
-  * [...]
-
-### Configuration
-
-The connector takes the following input:
-
-```yaml
-twitter:
-  - API-KEY
-  - Account List
-```
-
-### Output
-
-The connector will return the following:
-
-
-## Local development
-
-### Prerequisites
-
-#### Activate Virtual Environment and install dependencies
-From this connector directory, create a virtual environment:
-```
-python -m venv .venv
-```
-```
-source .venv/bin/activate
-pip install -r requirements.txt
-```
-
-### Locally running the connector
-```
-python main.py spec
-python main.py check --config sample_files/coin_list.json
-python main.py discover --config sample_files/coin_list.json
-python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json
-```
-
-### Locally running the connector docker image
-
-```bash
-docker build -t airbyte/social-network-fetcher:dev .
-# Running the spec command against your patched connector
-docker run airbyte/social-network-fetcher:dev spec
-````
-
-#### Run
-Then run any of the connector commands as follows:
-```
-docker run --rm airbyte/social-network-fetcher:dev spec
-docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev check --config /sample_files/coin_list.json
-docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev discover --config /sample_files/coin_list.json
-docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev read --config /sample_files/coin_list.json --catalog /sample_files/configured_catalog.json
-```

source-social-network-fetcher/source_social_network_fetcher/__init__.py

@@ -1,8 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-from .source import SourceSocialNetworkFetcher
-
-__all__ = ["SourceSocialNetworkFetcher"]

source-twitter-fetcher/README.md

@@ -0,0 +1,70 @@
+# Twitter Fetcher Source
+
+This is the repository for fetching Twitter information, written in Python.
+
+## Todos
+
+* [ ] Implements first version based on original script
+  * [ ] Fetch the Data users of each count.
+  * [ ] Fetch Tweets details 
+* [ ] Improve version:
+  * limit the data fetching based on input date
+
+
+## Usage
+
+This connector fetch information from Twitter based on their API: https://developer.twitter.com/en/docs/twitter-api
+
+### Configuration
+
+The connector takes the following input:
+
+```yaml
+twitter:
+  - API-KEY
+  - Account List
+```
+
+### Output
+
+The connector will return the following:
+
+
+## Local development
+
+### Prerequisites
+
+#### Activate Virtual Environment and install dependencies
+From this connector directory, create a virtual environment:
+```
+python -m venv .venv
+```
+```
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Locally running the connector
+```
+python main.py spec
+python main.py check --config sample_files/config-example.json
+python main.py discover --config sample_files/config-example.json
+python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json
+```
+
+### Locally running the connector docker image
+
+```bash
+docker build -t airbyte/twitter-fetcher:dev .
+# Running the spec command against your patched connector
+docker run airbyte/twitter-fetcher:dev spec
+````
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/twitter-fetcher:dev spec
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev check --config /sample_files/config-example.json
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev discover --config /sample_files/config-example.json
+docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev read --config /sample_files/config-example.json --catalog /sample_files/configured_catalog.json
+```

source-twitter-fetcher/main.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from source_social_network_fetcher.run import run
+from source_twitter_fetcher.run import run
 
 if __name__ == "__main__":
     run()

source-twitter-fetcher/metadata.yaml

@@ -10,16 +10,16 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
-  dockerImageTag: '0.0.3'
-  dockerRepository: status-im/airbyte/social-network-fetcher
-  githubIssueLabel: source-social-network-fetcher
-  icon: social-network-fetcher.svg
+  dockerImageTag: '0.0.1'
+  dockerRepository: status-im/airbyte/source-twitter-fetcher
+  githubIssueLabel: source-twitter-fetcher
+  icon: twitter-fetcher.svg
   license: MIT
-  name: Social Network Fetcher
+  name: Twitter Fetcher
   releaseDate: TODO
   supportLevel: community
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.com/integrations/sources/social-network-fetcher
+  documentationUrl: https://docs.airbyte.com/integrations/sources/twitter-fetcher
   tags:
     - language:python
 metadataSpecVersion: "1.0"

source-twitter-fetcher/setup.py

@@ -17,8 +17,8 @@ TEST_REQUIREMENTS = [
 ]
 
 setup(
-    name="source_social_network_fetcher",
-    description="Source implementation for Social Network Fetcher.",
+    name="source_twitter_fetcher",
+    description="Source implementation for Twitter Fetcher.",
     author="Status",
     author_email="devops@status.im",
     packages=find_packages(),
@@ -29,7 +29,7 @@ setup(
     },
     entry_points={
         "console_scripts": [
-            "source-social-network-fetcher=source_social_network_fetcher.run:run",
+            "source-twitter-fetcher=source_twitter_fetcher.run:run",
         ],
     },
 )

source-twitter-fetcher/source_twitter_fetcher/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from .source import SourceTwitterFetcher
+
+__all__ = ["SourceTwitterFetcher"]

source-twitter-fetcher/source_twitter_fetcher/run.py

@@ -6,8 +6,8 @@
 import sys
 
 from airbyte_cdk.entrypoint import launch
-from .source import SourceSocialNetworkFetcher
+from .source import SourceTwitterFetcher
 
 def run():
-    source = SourceSocialNetworkFetcher()
+    source = SourceTwitterFetcher()
     launch(source, sys.argv[1:])

source-twitter-fetcher/source_twitter_fetcher/source.py

@@ -7,6 +7,7 @@ from abc import ABC
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
 import logging
 import requests
+import time
 from datetime import datetime 
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
@@ -120,7 +121,7 @@ class TwitterTweet(HttpSubStream, TwitterAccountData):
     time.sleep(2)
 
 # Source
-class SourceSocialNetworkFetcher(AbstractSource):
+class SourceTwitterFetcher(AbstractSource):
   def check_connection(self, logger, config) -> Tuple[bool, any]:
     return True, None
 

source-twitter-fetcher/source_twitter_fetcher/spec.yaml

@@ -1,21 +1,23 @@
 documentationUrl: https://docsurl.com
 connectionSpecification:
   $schema: http://json-schema.org/draft-07/schema#
-  title: Social Network Fetcher
+  title: Twitter Fetcher
   type: object
   required:
     - api_key
     - accounts
+    - start_time
   properties:
     api_key:
       type: string
-      description: API Key to authentify to twitter
+      description: "API Key to authentify to twitter"
       airbyte_secret: true
     accounts:
       type: array
-      description: List of accounts needing to be extracted
+      description: "List of accounts needing to be extracted"
       items:
         type: string
     start_time:
       type: string
-      description: Start date of fetching data
+      description: "Start date of fetching data"
+      format: date-time