social-network-fetcher: Init connector

Signed-off-by: Alexis Pentori <alexis@status.im>

source-social-network-fetcher/Dockerfile

@@ -0,0 +1,8 @@
+FROM airbyte/python-connector-base:1.1.0
+
+COPY . ./airbyte/integration_code
+RUN pip install ./airbyte/integration_code
+
+# The entrypoint and default env vars are already set in the base image
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

source-social-network-fetcher/README.md

@@ -0,0 +1,74 @@
+# Social Network Fetcher Source
+
+This is the repository for fetching Social Network information, written in Python.
+
+## Todos
+
+* [ ] Implements first version based on original script
+  * [ ] Fetch the Data users of each count.
+  * [ ] Fetch Tweets details 
+* [ ] Improve version:
+  * limit the data fetching based on input date
+
+
+## Usage
+
+This connector fetch information from different social network based on their API:
+
+* Twitter:
+  * Tweets reaction
+  * [...]
+
+### Configuration
+
+The connector takes the following input:
+
+```yaml
+twitter:
+  - API-KEY
+  - Account List
+```
+
+### Output
+
+The connector will return the following:
+
+
+## Local development
+
+### Prerequisites
+
+#### Activate Virtual Environment and install dependencies
+From this connector directory, create a virtual environment:
+```
+python -m venv .venv
+```
+```
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Locally running the connector
+```
+python main.py spec
+python main.py check --config sample_files/coin_list.json
+python main.py discover --config sample_files/coin_list.json
+python main.py read --config sample_files/coin_list.json --catalog sample_files/configured_catalog.json
+```
+
+### Locally running the connector docker image
+
+```bash
+docker build -t airbyte/social-network-fetcher:dev .
+# Running the spec command against your patched connector
+docker run airbyte/social-network-fetcher:dev spec
+````
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/social-network-fetcher:dev spec
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev check --config /sample_files/coin_list.json
+docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev discover --config /sample_files/coin_list.json
+docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/social-network-fetcher:dev read --config /sample_files/coin_list.json --catalog /sample_files/configured_catalog.json
+```

source-social-network-fetcher/main.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from source_social_network_fetcher.run import run
+
+if __name__ == "__main__":
+    run()

source-social-network-fetcher/metadata.yaml

@@ -0,0 +1,25 @@
+data:
+  allowedHosts:
+  registries:
+    oss:
+      enabled: false
+    cloud:
+      enabled: false
+  connectorBuildOptions:
+    baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27
+  connectorSubtype: api
+  connectorType: source
+  definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
+  dockerImageTag: 0.0.2
+  dockerRepository: harbor.status.im/status-im/airbyte/source-social-network-fetcher
+  githubIssueLabel: source-social-network-fetcher
+  icon: social-network-fetcher.svg
+  license: MIT
+  name: Social Network Fetcher
+  releaseDate: TODO
+  supportLevel: community
+  releaseStage: alpha
+  documentationUrl: https://docs.airbyte.com/integrations/sources/social-network-fetcher
+  tags:
+    - language:python
+metadataSpecVersion: "1.0"

source-social-network-fetcher/requirements.txt

@@ -0,0 +1 @@
+-e .

source-social-network-fetcher/sample_files/config-example.json

@@ -0,0 +1,15 @@
+{
+  "api_key": "some_key",
+  "accounts": [
+    "Logos_network",
+    "Codex_storage",
+    "Waku_org",
+    "ethnimbus",
+    "ac1d_info",
+    "HashingItOutPod",
+    "vacp2p",
+    "InstituteFT"
+  ],
+  "start_time": "2024-01-01",
+  "stop_time": "2024-01-26"
+}

source-social-network-fetcher/sample_files/configured_catalog.json

@@ -0,0 +1,33 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "twitter_account_data",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "type": "object"
+        },
+        "supported_sync_modes": [
+          "full_refresh",  "incremental"
+        ]
+      },
+      "sync_mode":  "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "twitter_tweet",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "type": "object"
+        },
+        "supported_sync_modes": [
+          "full_refresh",  "incremental"
+        ]
+      },
+      "sync_mode":  "incremental",
+      "destination_sync_mode": "overwrite"
+    }
+
+  ]
+}

source-social-network-fetcher/setup.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from setuptools import find_packages, setup
+
+MAIN_REQUIREMENTS = [
+    "airbyte-cdk~=0.2",
+]
+
+TEST_REQUIREMENTS = [
+    "requests-mock~=1.9.3",
+    "pytest~=6.2",
+    "pytest-mock~=3.6.1",
+    "connector-acceptance-test",
+]
+
+setup(
+    name="source_social_network_fetcher",
+    description="Source implementation for Social Network Fetcher.",
+    author="Status",
+    author_email="devops@status.im",
+    packages=find_packages(),
+    install_requires=MAIN_REQUIREMENTS,
+    package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
+    entry_points={
+        "console_scripts": [
+            "source-social-network-fetcher=source_social_network_fetcher.run:run",
+        ],
+    },
+)

source-social-network-fetcher/source_social_network_fetcher/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from .source import SourceSocialNetworkFetcher
+
+__all__ = ["SourceSocialNetworkFetcher"]

source-social-network-fetcher/source_social_network_fetcher/run.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+
+from airbyte_cdk.entrypoint import launch
+from .source import SourceSocialNetworkFetcher
+
+def run():
+    source = SourceSocialNetworkFetcher()
+    launch(source, sys.argv[1:])

source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_account_data.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "account_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "account_name": {
+      "type": [
+      "null",
+      "string"
+      ]
+    },
+    "username": {
+        "type": [
+        "null",
+        "string"
+      ]
+    },
+    "tweet_count": {
+        "type": [
+        "null",
+        "number"
+      ]
+    },
+    "like_count": {
+        "type": [
+        "null",
+        "number"
+      ]
+    },
+    "following_count": {
+        "type": [
+        "null",
+        "number"
+      ]
+    },
+    "follower_count": {
+        "type": [
+        "null",
+        "number"
+      ]
+    },
+    "listed_count": {
+        "type": [
+        "null",
+        "number"
+      ]
+    }
+  }
+}

source-social-network-fetcher/source_social_network_fetcher/schemas/twitter_tweet.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "id": {
+      "created_at": [
+        "null",
+        "string"
+      ]
+    },
+    "retweet_count": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "reply_count": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "like_count": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "quote_count": {
+      "type": [
+        "null",
+        "number"
+      ]
+    },
+    "referenced_tweets": {
+      "type": [
+        "null",
+        "string"
+      ]
+    }
+  }
+}

source-social-network-fetcher/source_social_network_fetcher/source.py

@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from abc import ABC
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
+import logging
+import requests
+from datetime import datetime 
+from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
+from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+
+logger = logging.getLogger("airbyte")
+
+class TwitterStream(HttpStream):
+
+  url_base = "https://api.twitter.com/2/"
+
+  def __init__(self, api_key: str=None, accounts: List=None, start_time: str = None, stop_time: str = None, **kwargs):
+    super().__init__(**kwargs)
+    self.api_key = api_key
+    self.accounts = accounts
+    self.start_time = start_time
+    self.stop_time = stop_time;
+
+  def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+    return None
+
+class TwitterAccountData(TwitterStream):
+
+  primary_key = "account_id"
+
+  @property
+  def use_cache(self) -> bool:
+    return True
+
+
+  def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
+    for account in self.accounts:
+      yield {
+        "name": account
+      }
+
+
+  def path(
+    self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+  ) -> str:
+    return f"users/by/username/{stream_slice['name']}?user.fields=public_metrics"
+
+  def request_headers(
+    self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
+  ) -> MutableMapping[str, Any]:
+    return { 
+      "Authorization" : f"Bearer {self.api_key}",
+      "User-Agent": "v2RecentSearchPython"
+    }
+
+  def parse_response(
+        self, 
+        response: requests.Response,
+        stream_slice: Mapping[str, Any] = None,
+        **kwargs
+  ) -> Iterable[Mapping]:
+    logger.info("Getting data of %s account", stream_slice['name'])
+    logger.info("Response: %s", response.json())
+    data=response.json()['data']
+    yield {
+        "account_id": data['id'],
+        "username": data['username'],
+        "account_name": data['name'],
+        "tweet_count": data['public_metrics']['tweet_count'],
+        "like_count": data['public_metrics']['like_count'],
+        "following_count": data['public_metrics']['following_count'],
+        "follower_count": data['public_metrics']['followers_count'],
+        "listed_count": data['public_metrics']['listed_count'],
+        }
+
+class TwitterTweet(HttpSubStream, TwitterAccountData):
+    #TODO: See how to get the account ID
+  primary_key = ""
+  def __init__(self, **kwargs):
+    super().__init__(TwitterAccountData(**kwargs),**kwargs)
+
+  def path(
+    self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+  ) -> str:
+        account_id = stream_slice.get("parent").get("account_id")
+        return f"users/{account_id}/tweets?tweet.fields=text,public_metrics,author_id,referenced_tweets,created_at&start_time={self.start_time}"
+
+  def request_headers(
+    self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
+  ) -> MutableMapping[str, Any]:
+    return { 
+      "Authorization" : f"Bearer {self.api_key}",
+      "User-Agent": "v2RecentSearchPython"
+    }
+
+  def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+    data=response.json()['data']
+    logger.info("Response: %s", response.json())
+    referenced_tweets=""
+    for t in data:
+        if "referenced_tweets" in t:
+            for rt in t.get('referenced_tweets'):
+                referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
+        yield {
+                "id": t['id'],
+                "created_at": t.get('created_at'),
+                "retweet_count": t.get('public_metrics').get('retweet_count'),
+                "reply_count": t.get('public_metrics').get('reply_count'),
+                "like_count": t.get('public_metrics').get('like_count'),
+                "quote_count": t.get('public_metrics').get('quote_count'),
+                "referenced_tweets": referenced_tweets
+            }
+
+
+
+# Source
+class SourceSocialNetworkFetcher(AbstractSource):
+  def check_connection(self, logger, config) -> Tuple[bool, any]:
+    return True, None
+
+  def streams(self, config: Mapping[str, Any]) -> List[Stream]:       
+    return [
+      TwitterAccountData(
+        api_key=config['api_key'],
+        accounts=config['accounts']),
+      TwitterTweet(
+        api_key=config['api_key'],
+        accounts=config['accounts'],
+        start_time=config['start_time'], 
+        stop_time=datetime.now().isoformat())
+    ]

source-social-network-fetcher/source_social_network_fetcher/spec.yaml

@@ -0,0 +1,18 @@
+documentationUrl: https://docsurl.com
+connectionSpecification:
+  $schema: http://json-schema.org/draft-07/schema#
+  title: Social Network Fetcher
+  type: object
+  required:
+    - api_key
+    - accounts
+  properties:
+    api_key:
+      type: string
+      description: API Key to authentify to twitter
+      airbyte_secret: true
+    accounts:
+      type: array
+      description: List of accounts needing to be extracted
+      items:
+        type: string