simplecast-connector: init connector
Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
Alexis Pentori
parent
bf464bbed0
commit
efdc754e66
|
|
@ -0,0 +1,8 @@
|
|||
FROM airbyte/python-connector-base:1.1.0
|
||||
|
||||
COPY . ./airbyte/integration_code
|
||||
RUN pip install ./airbyte/integration_code
|
||||
|
||||
# The entrypoint and default env vars are already set in the base image
|
||||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# SimpleCast Fetcher Source
|
||||
|
||||
This is the repository for fetching SimpleCast data, written in Python.
|
||||
|
||||
## Usage
|
||||
|
||||
The connector fetch the list of podcasts, episodes and analytics metrics from [SimpleCast](https://www.simplecast.com/).
|
||||
|
||||
### Configuration
|
||||
|
||||
The connector takes the following input:
|
||||
|
||||
```yaml
|
||||
- api_key
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
The connector will return the following objects:
|
||||
- [podcast](./source_simplecast_fecther/schemas/podcast.json)
|
||||
- [episode](./source_simplecast_fecther/schemas/episode.json)
|
||||
|
||||
## Local development
|
||||
|
||||
### Prerequisites
|
||||
|
||||
#### Activate Virtual Environment and install dependencies
|
||||
From this connector directory, create a virtual environment:
|
||||
```
|
||||
python -m venv .venv
|
||||
```
|
||||
```
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Locally running the connector
|
||||
```
|
||||
python main.py spec
|
||||
python main.py check --config sample_files/config-example.json
|
||||
python main.py discover --config sample_files/config-example.json
|
||||
python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json
|
||||
```
|
||||
|
||||
### Locally running the connector docker image
|
||||
|
||||
```bash
|
||||
docker build -t airbyte/twitter-fetcher:dev .
|
||||
# Running the spec command against your patched connector
|
||||
docker run airbyte/twitter-fetcher:dev spec
|
||||
````
|
||||
|
||||
#### Run
|
||||
Then run any of the connector commands as follows:
|
||||
```
|
||||
docker run --rm airbyte/twitter-fetcher:dev spec
|
||||
docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev check --config /sample_files/config-example.json
|
||||
docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev discover --config /sample_files/config-example.json
|
||||
docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev read --config /sample_files/config-example.json --catalog /sample_files/configured_catalog.json
|
||||
```
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
from source_simplecast_fecther.run import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
data:
|
||||
allowedHosts:
|
||||
registries:
|
||||
oss:
|
||||
enabled: true
|
||||
cloud:
|
||||
enabled: false
|
||||
remoteRegistries:
|
||||
pypi:
|
||||
enabled: true
|
||||
packageName: airbyte-source-simplecast-fecther
|
||||
connectorBuildOptions:
|
||||
baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27
|
||||
connectorSubtype: api
|
||||
connectorType: source
|
||||
definitionId: 464a7cea-0317-485e-9a9c-bcd06155bfff
|
||||
dockerImageTag: 0.1.0
|
||||
dockerRepository: harbor.status.im/status-im/airbyte/source-simplecast-fecther
|
||||
githubIssueLabel: source-simplecast-fecther
|
||||
icon: simplecast-fecther.svg
|
||||
license: MIT
|
||||
name: Simplecast Fecther
|
||||
releaseDate: TODO
|
||||
supportLevel: community
|
||||
releaseStage: alpha
|
||||
documentationUrl: https://docs.airbyte.com/integrations/sources/simplecast-fecther
|
||||
tags:
|
||||
- language:python
|
||||
metadataSpecVersion: "1.0"
|
||||
|
|
@ -0,0 +1 @@
|
|||
-e .
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"api_key": "not-a-real-token"
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
{
|
||||
"streams": [
|
||||
{
|
||||
"stream": {
|
||||
"name": "podcast",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "episode",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "analytic_location",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "analytic_time_of_week",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "analytic_episode",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
},
|
||||
{
|
||||
"stream": {
|
||||
"name": "analytic_download",
|
||||
"json_schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"type": "object"
|
||||
},
|
||||
"supported_sync_modes": [
|
||||
"full_refresh", "incremental"
|
||||
]
|
||||
},
|
||||
"sync_mode": "incremental",
|
||||
"destination_sync_mode": "overwrite"
|
||||
}
|
||||
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
MAIN_REQUIREMENTS = [
|
||||
"airbyte-cdk~=0.2",
|
||||
]
|
||||
|
||||
TEST_REQUIREMENTS = [
|
||||
"requests-mock~=1.9.3",
|
||||
"pytest~=6.2",
|
||||
"pytest-mock~=3.6.1",
|
||||
"connector-acceptance-test",
|
||||
]
|
||||
|
||||
setup(
|
||||
name="source_simplecast_fecther",
|
||||
description="Source implementation for Simplecast Fecther.",
|
||||
author="Airbyte",
|
||||
author_email="contact@airbyte.io",
|
||||
packages=find_packages(),
|
||||
install_requires=MAIN_REQUIREMENTS,
|
||||
package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
|
||||
extras_require={
|
||||
"tests": TEST_REQUIREMENTS,
|
||||
},
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"source-simplecast-fecther=source_simplecast_fecther.run:run",
|
||||
],
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<svg width="176" height="32" viewBox="0 0 176 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_1749_53)">
|
||||
<path d="M53.2467 11.2389C51.3692 11.2389 50.5304 11.7462 50.5304 12.5267C50.5304 13.2682 51.1296 13.5414 52.1682 13.6584L56.9216 14.2048C59.6379 14.517 61.2357 15.6097 61.2357 18.1853C61.2357 21.6975 58.1599 22.556 53.4464 22.556C48.9725 22.556 45.9767 21.6584 45.8169 17.756H50.131C50.2508 19.5121 51.6489 19.8633 53.6062 19.8633C55.6833 19.8633 56.7618 19.395 56.7618 18.4194C56.7618 17.4828 55.9629 17.2097 54.8445 17.0536L50.5304 16.5072C47.8142 16.156 46.1764 15.1414 46.1764 12.6828C46.1764 9.48282 49.2122 8.5072 53.3665 8.5072C57.4808 8.5072 60.6365 9.44379 60.7563 12.8779H56.4423C56.3624 11.6292 55.0841 11.2389 53.2467 11.2389Z" fill="#0F0F09"/>
|
||||
<path d="M62.7539 8.74182H66.5487V10.9272H62.7539V8.74182ZM62.7938 12.215H66.5087V22.2052H62.7938V12.215Z" fill="#0F0F09"/>
|
||||
<path d="M72.141 13.5023H72.2609C73.1796 12.4096 74.8573 11.9413 76.3752 11.9413C78.0529 11.9413 79.3711 12.3315 80.0901 13.5023C81.1686 12.3706 82.8463 11.9413 84.5639 11.9413C86.9207 11.9413 88.7581 13.112 88.7581 15.7266V22.1657H85.0433V16.6242C85.0433 15.3364 84.2444 14.8291 83.0061 14.8291C81.8876 14.8291 80.5295 15.3364 80.5295 17.1705V22.1657H76.8146V16.6242C76.8146 15.3364 76.0157 14.8291 74.7774 14.8291C73.6589 14.8291 72.3008 15.3364 72.3008 17.1705V22.1657H68.5859V12.1754H72.2209V13.5023H72.141Z" fill="#0F0F09"/>
|
||||
<path d="M94.4302 21.19V25.5607H90.7153V12.2144H94.3503V13.5022H94.4702C95.3489 12.5656 96.6671 11.9802 98.3448 11.9802C101.82 11.9802 104.097 13.8534 104.097 17.2875C104.097 20.7217 101.82 22.5949 98.3448 22.5949C96.6671 22.5949 95.3889 22.0485 94.55 21.19H94.4302ZM97.3062 14.6339C95.3889 14.6339 94.3104 15.6485 94.3104 17.2875C94.3104 18.9266 95.3489 19.9412 97.3062 19.9412C99.2236 19.9412 100.222 19.0046 100.222 17.2875C100.222 15.5705 99.2236 14.6339 97.3062 14.6339Z" fill="#0F0F09"/>
|
||||
<path d="M105.655 8.81934H109.37V22.2047H105.655V8.81934Z" fill="#0F0F09"/>
|
||||
<path d="M124.469 17.2096C124.469 17.4437 124.469 17.834 124.429 18.1072H114.842C115.042 19.5511 116 20.2145 117.758 20.2145C119.675 20.2145 120.195 19.434 120.314 19.1218H124.349C124.189 20.7998 122.831 22.595 117.758 22.595C113.244 22.595 110.967 20.6047 110.967 17.2096C110.967 13.8145 113.244 11.8242 117.718 11.8242C122.192 11.8242 124.469 13.8145 124.469 17.2096ZM117.718 14.2047C116.2 14.2047 115.241 14.712 114.922 15.8437H120.514C120.155 14.7511 119.236 14.2047 117.718 14.2047Z" fill="#0F0F09"/>
|
||||
<path d="M135.014 18.3023H139.088C138.769 20.7609 137.051 22.556 132.258 22.556C127.864 22.556 125.587 20.5658 125.587 17.1706C125.587 13.7755 127.864 11.7853 132.258 11.7853C137.051 11.7853 138.729 13.5804 139.048 15.9999H134.974C134.854 15.2975 134.215 14.478 132.258 14.478C130.42 14.478 129.422 15.4926 129.422 17.1706C129.422 18.8877 130.42 19.8633 132.258 19.8633C134.255 19.9023 134.894 19.0828 135.014 18.3023Z" fill="#0F0F09"/>
|
||||
<path d="M153.149 22.0876V22.2047H149.514C149.354 22.0486 149.235 21.6584 149.235 21.1901V20.8389C148.236 22.0486 146.478 22.595 144.521 22.595C142.364 22.595 140.207 21.6193 140.207 19.3169C140.207 17.0925 141.565 16.312 143.562 16.1169L147.876 15.6876C148.635 15.6096 148.995 15.4535 148.995 14.9462C148.995 14.4779 148.396 14.1267 146.958 14.1267C145.839 14.1267 144.921 14.4389 144.881 15.1803H140.966C141.086 12.6047 143.802 11.8242 146.958 11.8242C150.433 11.8242 152.91 12.4876 152.91 14.9462V20.5657C152.87 21.1511 152.87 21.7754 153.149 22.0876ZM149.115 17.4437C148.875 17.5998 148.476 17.6779 148.116 17.7169L145.4 18.1072C144.561 18.2242 144.042 18.5364 144.042 19.2779C144.042 20.0974 144.721 20.3315 145.719 20.3315C147.078 20.3315 149.115 19.7072 149.115 17.834V17.4437Z" fill="#0F0F09"/>
|
||||
<path d="M162.816 15.1804C162.736 14.3609 161.737 14.0877 160.379 14.0877C158.981 14.0877 158.422 14.3999 158.422 14.8682C158.422 15.2975 158.702 15.4926 159.58 15.5706L163.335 15.9999C165.852 16.2731 167.17 16.9365 167.17 19.1999C167.17 21.8926 164.813 22.595 160.659 22.595C156.904 22.595 154.268 21.9316 154.148 19.0048H158.182C158.262 19.9804 159.101 20.2926 160.739 20.2926C162.496 20.2926 163.056 19.9414 163.056 19.395C163.056 18.8877 162.696 18.7316 161.817 18.6145L158.342 18.1853C155.906 17.8731 154.468 17.2487 154.468 15.1414C154.468 12.6048 157.024 11.9023 160.419 11.9023C163.934 11.9023 166.651 12.6828 166.731 15.2584L162.816 15.1804Z" fill="#0F0F09"/>
|
||||
<path d="M174.2 19.434C174.56 19.434 175.159 19.4341 175.638 19.356V22.1658C175.199 22.2438 174.2 22.3219 173.361 22.3219C170.246 22.3219 169.247 21.5023 169.247 19.2389V14.4389H167.529V12.2145H169.247V9.52185H172.962V12.2145H175.678V14.4389H172.962V18.5365C172.962 19.2389 173.441 19.434 174.2 19.434Z" fill="#0F0F09"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M0 0V32H32.7548V0H0ZM30.5978 29.8927H2.19697V2.14634H30.5978V29.8927Z" fill="#0F0F09"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M6.55127 6.3999V25.5999H26.2441L6.55127 6.3999Z" fill="#0F0F09"/>
|
||||
<path d="M26.2254 18.5166L13.8696 6.41516L12.3181 7.93475L24.6739 20.0362L26.2254 18.5166Z" fill="#0F0F09"/>
|
||||
<path d="M26.2526 12.4553L20.0747 6.40454L18.5232 7.92413L24.7011 13.9749L26.2526 12.4553Z" fill="#0F0F09"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_1749_53">
|
||||
<rect width="176" height="32" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.2 KiB |
|
|
@ -0,0 +1,8 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from .source import SourceSimplecastFecther
|
||||
|
||||
__all__ = ["SourceSimplecastFecther"]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
from airbyte_cdk.entrypoint import launch
|
||||
from .source import SourceSimplecastFecther
|
||||
|
||||
def run():
|
||||
source = SourceSimplecastFecther()
|
||||
launch(source, sys.argv[1:])
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"interval": {
|
||||
"name": ["null", "string"]
|
||||
},
|
||||
"downloads_total": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"downloads_percent": {
|
||||
"type": ["null", "number"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"name": ["null", "string"]
|
||||
},
|
||||
"type": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"title": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"downloads": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"number": {
|
||||
"type": ["null", "number"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"rank": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"name": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"downloads_total": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"downloads_percent": {
|
||||
"type": ["null", "number"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"rank": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"hour_of_week": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"hour_of_day": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"day_of_week": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"count": {
|
||||
"type": ["null", "count"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"title": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"published_at": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"updated_at": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"status": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"season_href": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"season_number": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"number": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"descritpion": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"duration": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"token": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"type": {
|
||||
"type": ["null", "string"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"title": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"status": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"href": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"episode_count": {
|
||||
"type": ["null", "number"]
|
||||
},
|
||||
"account_id": {
|
||||
"type": ["null", "string"]
|
||||
},
|
||||
"account_owner_name": {
|
||||
"type": ["null", "string"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
#
|
||||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||
#
|
||||
|
||||
|
||||
from abc import ABC
|
||||
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
|
||||
import logging
|
||||
import requests
|
||||
from airbyte_cdk.sources import AbstractSource
|
||||
from airbyte_cdk.sources.streams import Stream
|
||||
from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
|
||||
from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
|
||||
|
||||
logger = logging.getLogger("airbyte")
|
||||
|
||||
|
||||
LOCATION_KEYS=["id", "rank", "name", "downloads_total", "downloads_percent"]
|
||||
TIME_OF_WEEK_KEYS=["rank", "hour_of_week", "hour_of_day", "day_of_week", "count"]
|
||||
DOWNLOADS_KEY=["interval", "downloads_total", "downloads_percent"]
|
||||
# Basic full refresh stream
|
||||
class SimplecastFectherStream(HttpStream):
|
||||
url_base = "https://api.simplecast.com/"
|
||||
|
||||
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
||||
return None
|
||||
|
||||
class Podcast(SimplecastFectherStream):
|
||||
|
||||
primary_key = "podcast_id"
|
||||
|
||||
|
||||
@property
|
||||
def use_cache(self) -> bool:
|
||||
return True
|
||||
|
||||
def path(
|
||||
self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
return "podcasts"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.info("Response: %s", data)
|
||||
if 'collection' not in data.keys():
|
||||
logger.debug("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('collection'):
|
||||
podcast={
|
||||
"id": elt.get("id"),
|
||||
"title": elt.get("title"),
|
||||
"status": elt.get("status"),
|
||||
"href": elt.get("href"),
|
||||
"episode_count": elt.get("episodes").get("count"),
|
||||
"account_id": elt.get("account_id"),
|
||||
"account_owner_name": elt.get("account").get("owner").get("name")
|
||||
}
|
||||
yield podcast
|
||||
|
||||
class Episode(HttpSubStream, Podcast):
|
||||
primary_key="episode_id"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(Podcast(**kwargs), **kwargs)
|
||||
|
||||
def path(
|
||||
self,
|
||||
stream_state: Mapping[str, Any] = None,
|
||||
stream_slice: Mapping[str, Any] = None,
|
||||
next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
podcast_id=stream_slice.get("parent").get("id")
|
||||
|
||||
return f"podcasts/{podcast_id}/episodes"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.debug("Response: %s", data)
|
||||
if 'collection' not in data.keys():
|
||||
logger.error("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('collection'):
|
||||
episode={
|
||||
"id": elt.get("id"),
|
||||
"title": elt.get("title"),
|
||||
"status": elt.get("status"),
|
||||
"published_at": elt.get("published_at"),
|
||||
"updated_at": elt.get("updated_at"),
|
||||
"season_href": elt.get('season').get("href"),
|
||||
"season_number": elt.get('season').get("number"),
|
||||
"number": elt.get("number"),
|
||||
"description": elt.get("description"),
|
||||
"token": elt.get("token"),
|
||||
"type": elt.get("type")
|
||||
}
|
||||
yield episode
|
||||
|
||||
class AnalyticLocation(HttpSubStream, Podcast):
|
||||
primary_key="analytic_location_id"
|
||||
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(Podcast(**kwargs), **kwargs)
|
||||
|
||||
def path(
|
||||
self,
|
||||
stream_state: Mapping[str, Any] = None,
|
||||
stream_slice: Mapping[str, Any] = None,
|
||||
next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
podcast_id=stream_slice.get("parent").get("id")
|
||||
|
||||
return f"analytics/location?podcast={podcast_id}"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.info("Response: %s", data)
|
||||
if 'countries' not in data.keys():
|
||||
logger.error("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('countries'):
|
||||
location={ key: elt.get(key) for key in LOCATION_KEYS }
|
||||
yield location
|
||||
|
||||
class AnalyticTimeOfWeek(HttpSubStream, Podcast):
|
||||
primary_key=None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(Podcast(**kwargs), **kwargs)
|
||||
|
||||
def path(
|
||||
self,
|
||||
stream_state: Mapping[str, Any] = None,
|
||||
stream_slice: Mapping[str, Any] = None,
|
||||
next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
podcast_id=stream_slice.get("parent").get("id")
|
||||
|
||||
return f"analytics/time_of_week?podcast={podcast_id}"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.info("Response: %s", data)
|
||||
if 'collection' not in data.keys():
|
||||
logger.error("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('collection'):
|
||||
time_of_week={ key: elt.get(key) for key in TIME_OF_WEEK_KEYS }
|
||||
yield time_of_week
|
||||
|
||||
class AnalyticEpisode(HttpSubStream, Podcast):
|
||||
primary_key=None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(Podcast(**kwargs), **kwargs)
|
||||
|
||||
def path(
|
||||
self,
|
||||
stream_state: Mapping[str, Any] = None,
|
||||
stream_slice: Mapping[str, Any] = None,
|
||||
next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
podcast_id=stream_slice.get("parent").get("id")
|
||||
|
||||
return f"analytics/episodes?podcast={podcast_id}"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.info("Response: %s", data)
|
||||
if 'collection' not in data.keys():
|
||||
logger.error("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('collection'):
|
||||
analytic_episode={
|
||||
"id": elt.get("id"),
|
||||
"type": elt.get("type"),
|
||||
"title": elt.get("title"),
|
||||
"downloads": elt.get("downloads").get("total"),
|
||||
"number": elt.get("number")
|
||||
}
|
||||
yield analytic_episode
|
||||
|
||||
class AnalyticDownload(HttpSubStream, Podcast):
|
||||
primary_key=None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(Podcast(**kwargs), **kwargs)
|
||||
|
||||
def path(
|
||||
self,
|
||||
stream_state: Mapping[str, Any] = None,
|
||||
stream_slice: Mapping[str, Any] = None,
|
||||
next_page_token: Mapping[str, Any] = None
|
||||
) -> str:
|
||||
podcast_id=stream_slice.get("parent").get("id")
|
||||
|
||||
return f"analytics/downloads?podcast={podcast_id}"
|
||||
|
||||
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
|
||||
data=response.json()
|
||||
logger.info("Response: %s", data)
|
||||
if 'by_interval' not in data.keys():
|
||||
logger.error("Error when trying to get the data %s", data)
|
||||
raise Exception("error when calling the api")
|
||||
for elt in data.get('by_interval'):
|
||||
download={ key: elt.get(key) for key in DOWNLOADS_KEY }
|
||||
yield download
|
||||
|
||||
# Source
|
||||
class SourceSimplecastFecther(AbstractSource):
|
||||
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
||||
return True, None
|
||||
|
||||
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
||||
auth = TokenAuthenticator(token=config["api_key"])
|
||||
return [
|
||||
Podcast(authenticator=auth),
|
||||
Episode(authenticator=auth),
|
||||
AnalyticLocation(authenticator=auth),
|
||||
AnalyticTimeOfWeek(authenticator=auth),
|
||||
AnalyticEpisode(authenticator=auth),
|
||||
AnalyticDownload(authenticator=auth)
|
||||
]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
documentationUrl: https://docsurl.com
|
||||
connectionSpecification:
|
||||
$schema: http://json-schema.org/draft-07/schema#
|
||||
title: Simplecast Fecther Spec
|
||||
type: object
|
||||
required:
|
||||
- api_key
|
||||
properties:
|
||||
api_key:
|
||||
type: string
|
||||
description: Key to authentify to the API
|
||||
airbyte_secret: true
|
||||
Loading…
Reference in New Issue