From 083fa2d6a77343a99c8c35ab442777a6f985655a Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Tue, 23 Apr 2024 10:26:00 +0200 Subject: [PATCH] simplecast: adding analytic_episode_download Signed-off-by: Alexis Pentori --- source-simplecast-fecther/metadata.yaml | 2 +- .../sample_files/configured_catalog.json | 28 +++++++------- .../schemas/analytic_episode_download.json | 36 ++++++++++++++++++ .../source_simplecast_fecther/source.py | 37 ++++++++----------- 4 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_episode_download.json diff --git a/source-simplecast-fecther/metadata.yaml b/source-simplecast-fecther/metadata.yaml index c03f9dd..e4cdd70 100644 --- a/source-simplecast-fecther/metadata.yaml +++ b/source-simplecast-fecther/metadata.yaml @@ -14,7 +14,7 @@ data: connectorSubtype: api connectorType: source definitionId: 464a7cea-0317-485e-9a9c-bcd06155bfff - dockerImageTag: 1.1.1 + dockerImageTag: 1.1.2 dockerRepository: harbor.status.im/status-im/airbyte/source-simplecast-fetcher githubIssueLabel: source-simplecast-fecther icon: simplecast-fecther.svg diff --git a/source-simplecast-fecther/sample_files/configured_catalog.json b/source-simplecast-fecther/sample_files/configured_catalog.json index 33e3aaa..ce353f6 100644 --- a/source-simplecast-fecther/sample_files/configured_catalog.json +++ b/source-simplecast-fecther/sample_files/configured_catalog.json @@ -56,20 +56,6 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "analytic_episode", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": [ - "full_refresh", "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "analytic_download", @@ -153,6 +139,20 @@ }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "analytic_episode_download", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": [ + "full_refresh", "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" } ] } diff --git a/source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_episode_download.json b/source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_episode_download.json new file mode 100644 index 0000000..502003e --- /dev/null +++ b/source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_episode_download.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Generated schema for Root", + "type": "object", + "properties": { + "href": { + "type": ["null", "string"] + }, + "total": { + "type": ["null", "number"] + }, + "interval": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "by_interval": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "interval": { + "type": ["null", "string"] + }, + "downloads_total": { + "type": ["null", "number"] + }, + "downloads_percent": { + "type": ["null", "number"] + } + } + } + } + } +} diff --git a/source-simplecast-fecther/source_simplecast_fecther/source.py b/source-simplecast-fecther/source_simplecast_fecther/source.py index 16a43c5..919cfa0 100644 --- a/source-simplecast-fecther/source_simplecast_fecther/source.py +++ b/source-simplecast-fecther/source_simplecast_fecther/source.py @@ -152,25 +152,6 @@ class AnalyticTimeOfWeek(AnalyticSubStream): def __init__(self, **kwargs): super().__init__(endpoint="time_of_week", keys_dict=TIME_OF_WEEK_KEYS, collection_name="collection", **kwargs) -class AnalyticEpisode(AnalyticSubStream): - primary_key=None - - def __init__(self, **kwargs): - super().__init__(endpoint="episodes", keys_dict=[], collection_name="", **kwargs) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data=response.json() - logger.debug("Response: %s", data) - for elt in data.get('collection'): - analytic_episode={ - "id": elt.get("id"), - "type": elt.get("type"), - "title": elt.get("title"), - "downloads": elt.get("downloads").get("total"), - "number": elt.get("number") - } - yield analytic_episode - class AnalyticDownload(AnalyticSubStream): def __init__(self, **kwargs): @@ -221,8 +202,20 @@ class AnalyticPodcastV2(HttpSubStream, SimplecastFectherStream): logger.debug("Response: %s", data) yield data +class AnalyticEpisodeDownload(HttpSubStream, SimplecastFectherStream): + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: + episode_id=stream_slice.get("parent").get("id") + return f"analytics/downloads?episode={episode_id}" - + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data=response.json() + logger.debug("Response: %s", data) + yield data # Source class SourceSimplecastFecther(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -237,11 +230,11 @@ class SourceSimplecastFecther(AbstractSource): episodes, AnalyticLocation(authenticator=auth, parent=podcasts), AnalyticTimeOfWeek(authenticator=auth, parent=podcasts), - AnalyticEpisode(authenticator=auth, parent=podcasts), AnalyticDownload(authenticator=auth,parent=podcasts), TechnologyApplication(authenticator=auth, parent=podcasts), TechnologyDeviceClass(authenticator=auth, parent=podcasts), TechnologyListeningMethod(authenticator=auth, parent=podcasts), AnalyticEpisodeV2(authenticator=auth, parent=episodes), - AnalyticPodcastV2(authenticator=auth, parent=podcasts) + AnalyticPodcastV2(authenticator=auth, parent=podcasts), + AnalyticEpisodeDownload(authenticator=auth, parent=episodes) ]