discord: fetching role in guild
Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
parent
7315c8d734
commit
77f153e18e
|
@ -17,7 +17,7 @@ data:
|
||||||
connectorSubtype: api
|
connectorSubtype: api
|
||||||
connectorType: source
|
connectorType: source
|
||||||
definitionId: b62a9143-1b59-41b2-9942-bae709e7da6b
|
definitionId: b62a9143-1b59-41b2-9942-bae709e7da6b
|
||||||
dockerImageTag: 1.0.1
|
dockerImageTag: 1.1.0
|
||||||
dockerRepository: harbor.status.im/status-im/airbyte/source-discord-fetcher
|
dockerRepository: harbor.status.im/status-im/airbyte/source-discord-fetcher
|
||||||
githubIssueLabel: source-discord-fetcher
|
githubIssueLabel: source-discord-fetcher
|
||||||
icon: discord-fetcher.svg
|
icon: discord-fetcher.svg
|
||||||
|
|
|
@ -55,6 +55,20 @@
|
||||||
},
|
},
|
||||||
"sync_mode": "incremental",
|
"sync_mode": "incremental",
|
||||||
"destination_sync_mode": "overwrite"
|
"destination_sync_mode": "overwrite"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"stream": {
|
||||||
|
"name": "guild_role",
|
||||||
|
"json_schema": {
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"supported_sync_modes": [
|
||||||
|
"full_refresh", "incremental"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sync_mode": "incremental",
|
||||||
|
"destination_sync_mode": "overwrite"
|
||||||
}
|
}
|
||||||
|
|
||||||
]
|
]
|
||||||
|
|
|
@ -43,6 +43,9 @@
|
||||||
},
|
},
|
||||||
"total_message_sent":{
|
"total_message_sent":{
|
||||||
"type": ["null", "number"]
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"guild_id":{
|
||||||
|
"type": ["null", "number"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,67 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"number"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"string"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"number"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hoist": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"boolean"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"position": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"number"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"permissions": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"string"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"managed": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"boolean"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"mentionable": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"boolean"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"integer"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"guild_id": {
|
||||||
|
"type": [
|
||||||
|
"null",
|
||||||
|
"integer"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -67,6 +67,12 @@
|
||||||
"null",
|
"null",
|
||||||
"integer"
|
"integer"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"roles":{
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,7 @@ logger = logging.getLogger("airbyte")
|
||||||
GUILD_KEYS = ["id", "name", "owner_id", "roles", "description", "chain", "max_members"]
|
GUILD_KEYS = ["id", "name", "owner_id", "roles", "description", "chain", "max_members"]
|
||||||
CHANNEL_KEYS = ["id", "type", "guild_id", "position", "name", "topic", "last_message_id", "managed", "parent_id", "last_pin_timestamp", "message_count", "member_count", "falgs", "total_message_sent"]
|
CHANNEL_KEYS = ["id", "type", "guild_id", "position", "name", "topic", "last_message_id", "managed", "parent_id", "last_pin_timestamp", "message_count", "member_count", "falgs", "total_message_sent"]
|
||||||
USER_KEYS = [ "id", "username", "discriminator", "global_name", "bot", "mfa_enabled", "verified", "email", "premium_type", "public_flags"]
|
USER_KEYS = [ "id", "username", "discriminator", "global_name", "bot", "mfa_enabled", "verified", "email", "premium_type", "public_flags"]
|
||||||
|
ROLES_KEYS = ["id", "name", "color", "hoist", "position", "permissions", "managed", "mentionable", "falgs", "guild_id"]
|
||||||
|
|
||||||
MAX_USERS = 1000
|
MAX_USERS = 1000
|
||||||
# Basic full refresh stream
|
# Basic full refresh stream
|
||||||
|
@ -44,7 +45,7 @@ class DiscordFetcherStream(HttpStream, ABC):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
class Guild(DiscordFetcherStream):
|
class Guild(DiscordFetcherStream):
|
||||||
primary_key = "guild_id"
|
primary_key = "id"
|
||||||
|
|
||||||
def parse_response(
|
def parse_response(
|
||||||
self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs
|
self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs
|
||||||
|
@ -56,7 +57,7 @@ class Guild(DiscordFetcherStream):
|
||||||
|
|
||||||
|
|
||||||
class GuildChannel(DiscordFetcherStream):
|
class GuildChannel(DiscordFetcherStream):
|
||||||
primary_key="channel_id"
|
primary_key="id"
|
||||||
|
|
||||||
use_cache=True
|
use_cache=True
|
||||||
|
|
||||||
|
@ -71,7 +72,7 @@ class GuildChannel(DiscordFetcherStream):
|
||||||
|
|
||||||
|
|
||||||
class Channel(HttpSubStream, DiscordFetcherStream):
|
class Channel(HttpSubStream, DiscordFetcherStream):
|
||||||
primary_key="channel_id"
|
primary_key="id"
|
||||||
|
|
||||||
def path(
|
def path(
|
||||||
self,
|
self,
|
||||||
|
@ -92,7 +93,7 @@ class Channel(HttpSubStream, DiscordFetcherStream):
|
||||||
yield channel
|
yield channel
|
||||||
|
|
||||||
class Member(DiscordFetcherStream):
|
class Member(DiscordFetcherStream):
|
||||||
primary_key="member_id"
|
primary_key="id"
|
||||||
|
|
||||||
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
||||||
# if the response doesn't contain the maximum number of user then there is no more to fetch
|
# if the response doesn't contain the maximum number of user then there is no more to fetch
|
||||||
|
@ -121,8 +122,25 @@ class Member(DiscordFetcherStream):
|
||||||
for elt in data:
|
for elt in data:
|
||||||
user = { key : elt.get('user').get(key) for key in USER_KEYS }
|
user = { key : elt.get('user').get(key) for key in USER_KEYS }
|
||||||
user['guild_id']=stream_slice['guild_id']
|
user['guild_id']=stream_slice['guild_id']
|
||||||
|
user['roles']=elt.get('roles')
|
||||||
yield user
|
yield user
|
||||||
|
|
||||||
|
class GuildRole(DiscordFetcherStream):
|
||||||
|
primary_key=None
|
||||||
|
|
||||||
|
def parse_response(
|
||||||
|
self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs
|
||||||
|
) -> Iterable[Mapping]:
|
||||||
|
logger.debug("Response: %s", response.json())
|
||||||
|
data=response.json()
|
||||||
|
for elt in data:
|
||||||
|
logger.info('Role : %s', elt)
|
||||||
|
role = { key : elt.get(key) for key in ROLES_KEYS }
|
||||||
|
role['guild_id']=stream_slice['guild_id']
|
||||||
|
yield role
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Source
|
# Source
|
||||||
class SourceDiscordFetcher(AbstractSource):
|
class SourceDiscordFetcher(AbstractSource):
|
||||||
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
||||||
|
@ -135,5 +153,6 @@ class SourceDiscordFetcher(AbstractSource):
|
||||||
Guild(guilds_id=config["guilds_id"], authenticator=auth),
|
Guild(guilds_id=config["guilds_id"], authenticator=auth),
|
||||||
guildChannel,
|
guildChannel,
|
||||||
Channel(guilds_id=config["guilds_id"], authenticator=auth, parent=guildChannel),
|
Channel(guilds_id=config["guilds_id"], authenticator=auth, parent=guildChannel),
|
||||||
Member(guilds_id=config["guilds_id"], endpoint="/members", authenticator=auth)
|
Member(guilds_id=config["guilds_id"], endpoint="/members", authenticator=auth),
|
||||||
|
GuildRole(guilds_id=config["guilds_id"], endpoint="/roles", authenticator=auth)
|
||||||
]
|
]
|
||||||
|
|
Loading…
Reference in New Issue