diff --git a/source-discord-fetcher/metadata.yaml b/source-discord-fetcher/metadata.yaml index 048994c..c0c161b 100644 --- a/source-discord-fetcher/metadata.yaml +++ b/source-discord-fetcher/metadata.yaml @@ -17,7 +17,7 @@ data: connectorSubtype: api connectorType: source definitionId: b62a9143-1b59-41b2-9942-bae709e7da6b - dockerImageTag: 1.0.1 + dockerImageTag: 1.1.0 dockerRepository: harbor.status.im/status-im/airbyte/source-discord-fetcher githubIssueLabel: source-discord-fetcher icon: discord-fetcher.svg diff --git a/source-discord-fetcher/sample_files/configured_catalog.json b/source-discord-fetcher/sample_files/configured_catalog.json index a5e269c..a6d70c5 100644 --- a/source-discord-fetcher/sample_files/configured_catalog.json +++ b/source-discord-fetcher/sample_files/configured_catalog.json @@ -55,6 +55,20 @@ }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "guild_role", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object" + }, + "supported_sync_modes": [ + "full_refresh", "incremental" + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" } ] diff --git a/source-discord-fetcher/source_discord_fetcher/schemas/guild_channel.json b/source-discord-fetcher/source_discord_fetcher/schemas/guild_channel.json index 39b53a6..18aa259 100644 --- a/source-discord-fetcher/source_discord_fetcher/schemas/guild_channel.json +++ b/source-discord-fetcher/source_discord_fetcher/schemas/guild_channel.json @@ -43,6 +43,9 @@ }, "total_message_sent":{ "type": ["null", "number"] + }, + "guild_id":{ + "type": ["null", "number"] } } } diff --git a/source-discord-fetcher/source_discord_fetcher/schemas/guild_role.json b/source-discord-fetcher/source_discord_fetcher/schemas/guild_role.json new file mode 100644 index 0000000..c234e79 --- /dev/null +++ b/source-discord-fetcher/source_discord_fetcher/schemas/guild_role.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "number" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "color": { + "type": [ + "null", + "number" + ] + }, + "hoist": { + "type": [ + "null", + "boolean" + ] + }, + "position": { + "type": [ + "null", + "number" + ] + }, + "permissions": { + "type": [ + "null", + "string" + ] + }, + "managed": { + "type": [ + "null", + "boolean" + ] + }, + "mentionable": { + "type": [ + "null", + "boolean" + ] + }, + "flags": { + "type": [ + "null", + "integer" + ] + }, + "guild_id": { + "type": [ + "null", + "integer" + ] + } + + } +} diff --git a/source-discord-fetcher/source_discord_fetcher/schemas/member.json b/source-discord-fetcher/source_discord_fetcher/schemas/member.json index 37d07f6..f650a09 100644 --- a/source-discord-fetcher/source_discord_fetcher/schemas/member.json +++ b/source-discord-fetcher/source_discord_fetcher/schemas/member.json @@ -67,6 +67,12 @@ "null", "integer" ] + }, + "roles":{ + "type": "array", + "items": { + "type": ["null", "number"] + } } } } diff --git a/source-discord-fetcher/source_discord_fetcher/source.py b/source-discord-fetcher/source_discord_fetcher/source.py index 9a27fa8..baf73e6 100644 --- a/source-discord-fetcher/source_discord_fetcher/source.py +++ b/source-discord-fetcher/source_discord_fetcher/source.py @@ -15,6 +15,7 @@ logger = logging.getLogger("airbyte") GUILD_KEYS = ["id", "name", "owner_id", "roles", "description", "chain", "max_members"] CHANNEL_KEYS = ["id", "type", "guild_id", "position", "name", "topic", "last_message_id", "managed", "parent_id", "last_pin_timestamp", "message_count", "member_count", "falgs", "total_message_sent"] USER_KEYS = [ "id", "username", "discriminator", "global_name", "bot", "mfa_enabled", "verified", "email", "premium_type", "public_flags"] +ROLES_KEYS = ["id", "name", "color", "hoist", "position", "permissions", "managed", "mentionable", "falgs", "guild_id"] MAX_USERS = 1000 # Basic full refresh stream @@ -44,7 +45,7 @@ class DiscordFetcherStream(HttpStream, ABC): return None class Guild(DiscordFetcherStream): - primary_key = "guild_id" + primary_key = "id" def parse_response( self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs @@ -56,7 +57,7 @@ class Guild(DiscordFetcherStream): class GuildChannel(DiscordFetcherStream): - primary_key="channel_id" + primary_key="id" use_cache=True @@ -71,7 +72,7 @@ class GuildChannel(DiscordFetcherStream): class Channel(HttpSubStream, DiscordFetcherStream): - primary_key="channel_id" + primary_key="id" def path( self, @@ -92,7 +93,7 @@ class Channel(HttpSubStream, DiscordFetcherStream): yield channel class Member(DiscordFetcherStream): - primary_key="member_id" + primary_key="id" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: # if the response doesn't contain the maximum number of user then there is no more to fetch @@ -121,8 +122,25 @@ class Member(DiscordFetcherStream): for elt in data: user = { key : elt.get('user').get(key) for key in USER_KEYS } user['guild_id']=stream_slice['guild_id'] + user['roles']=elt.get('roles') yield user +class GuildRole(DiscordFetcherStream): + primary_key=None + + def parse_response( + self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + logger.debug("Response: %s", response.json()) + data=response.json() + for elt in data: + logger.info('Role : %s', elt) + role = { key : elt.get(key) for key in ROLES_KEYS } + role['guild_id']=stream_slice['guild_id'] + yield role + + + # Source class SourceDiscordFetcher(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -135,5 +153,6 @@ class SourceDiscordFetcher(AbstractSource): Guild(guilds_id=config["guilds_id"], authenticator=auth), guildChannel, Channel(guilds_id=config["guilds_id"], authenticator=auth, parent=guildChannel), - Member(guilds_id=config["guilds_id"], endpoint="/members", authenticator=auth) + Member(guilds_id=config["guilds_id"], endpoint="/members", authenticator=auth), + GuildRole(guilds_id=config["guilds_id"], endpoint="/roles", authenticator=auth) ]