diff --git a/src/plugins/discord/fetcher.js b/src/plugins/discord/fetcher.js index 0cad0e6..b715f10 100644 --- a/src/plugins/discord/fetcher.js +++ b/src/plugins/discord/fetcher.js @@ -25,6 +25,30 @@ export type ResultPage = {| +results: $ReadOnlyArray, |}; +/** + * An interface to fetch Discord data + */ +export interface DiscordFetcher { + channels(guildId: Snowflake): Promise<$ReadOnlyArray>; + + members( + guildId: Snowflake, + after: Snowflake + ): Promise>; + + messages( + channel: Snowflake, + after: Snowflake + ): Promise>; + + reactions( + channel: Snowflake, + message: Snowflake, + emoji: Model.Emoji, + after: Snowflake + ): Promise>; +} + /** * Fetcher is responsible for: * - Returning the correct endpoint to fetch against for Guilds, Channels, @@ -43,7 +67,7 @@ export type ResultPage = {| * returning an array of Channel objects in the corresponding method. * See: https://discordapp.com/developers/docs/resources/guild#get-guild-channels */ -export class Fetcher { +export class Fetcher implements DiscordFetcher { +_fetch: FetchEndpoint; +_options: FetchOptions; diff --git a/src/plugins/discord/fetcher.test.js b/src/plugins/discord/fetcher.test.js index 5f05637..0c10dc1 100644 --- a/src/plugins/discord/fetcher.test.js +++ b/src/plugins/discord/fetcher.test.js @@ -7,9 +7,9 @@ import { type Reaction, type Message, type GuildMember, - type Snowflake, type Emoji, } from "./models"; +import {testChannel} from "./testUtils"; /** * Note: the 'any' type signature is assigned to server response objects @@ -50,20 +50,14 @@ describe("plugins/discord/fetcher", () => { }); it("handles response", async () => { - const testChannel = (id: string): any => ({ + const testChannelResp = (id: string): any => ({ id: id, name: "testChannelName", type: 0, }); - const testChannelObj = (id: Snowflake): Channel => ({ - id: id, - name: "testChannelName", - type: "GUILD_TEXT", - }); - - const response: any[] = [testChannel("1")]; - const expected: Channel[] = [testChannelObj("1")]; + const response: any[] = [testChannelResp("1")]; + const expected: Channel[] = [testChannel("1")]; const fetch = jest.fn(() => Promise.resolve(response)); const fetcher = new Fetcher(fetch, defaultOptions()); const data = await fetcher.channels("0"); diff --git a/src/plugins/discord/mirror.js b/src/plugins/discord/mirror.js new file mode 100644 index 0000000..0319bc8 --- /dev/null +++ b/src/plugins/discord/mirror.js @@ -0,0 +1,78 @@ +// @flow + +import {type ResultPage} from "./fetcher"; +import {type SqliteMirror} from "./sqliteMirror"; +import {type Snowflake} from "./models"; +import * as Model from "./models"; +import * as nullUtil from "../../util/null"; + +/** + * Mirrors data from the Discord API into a local sqlite db. + */ + +export async function fetchDiscord( + sqliteMirror: SqliteMirror, + streams: DepaginatedFetcher +) { + for (const member of await streams.members()) { + sqliteMirror.addMember(member); + } + + for (const channel of await streams.channels()) { + sqliteMirror.addChannel(channel); + + for (const message of await streams.messages(channel.id)) { + sqliteMirror.addMessage(message); + + for (const emoji of message.reactionEmoji) { + for (const reaction of await streams.reactions( + channel.id, + message.id, + emoji + )) { + sqliteMirror.addReaction(reaction); + } + } + } + } +} + +// Note: most of this is about wrapping pagination. +export interface DepaginatedFetcher { + members(after: ?Snowflake): Promise<$ReadOnlyArray>; + channels(): Promise<$ReadOnlyArray>; + messages( + channel: Snowflake, + after: ?Snowflake + ): Promise<$ReadOnlyArray>; + reactions( + channel: Snowflake, + message: Snowflake, + emoji: Model.Emoji, + after: ?Snowflake + ): Promise<$ReadOnlyArray>; +} + +export async function getPages( + fetchPage: (endCursor: Snowflake) => Promise>, + endCursor: Snowflake +): Promise<$ReadOnlyArray> { + let after = endCursor; + let hasNextPage = true; + let allResults = []; + + while (hasNextPage) { + const {results, pageInfo} = await fetchPage(after); + allResults = allResults.concat(results); + + hasNextPage = pageInfo.hasNextPage; + if (hasNextPage) { + after = nullUtil.get( + pageInfo.endCursor, + `Found null endCursor with hasNextPage = true` + ); + } + } + + return allResults; +} diff --git a/src/plugins/discord/mirror.test.js b/src/plugins/discord/mirror.test.js new file mode 100644 index 0000000..c09ca12 --- /dev/null +++ b/src/plugins/discord/mirror.test.js @@ -0,0 +1,137 @@ +// @flow + +import Database from "better-sqlite3"; +import {fetchDiscord, DepaginatedFetcher, getPages} from "./mirror"; +import {SqliteMirror} from "./sqliteMirror"; +import * as Model from "./models"; +import {type Snowflake} from "./models"; +import { + testChannel, + testMember, + testMessage, + customEmoji, + testReaction, +} from "./testUtils"; + +describe("plugins/discord/mirror", () => { + describe("updateMirror", () => { + const activeChannelId = "12"; + const activeMessageid = "34"; + + function testMirror() { + const db = new Database(":memory:"); + return new SqliteMirror(db, "0"); + } + + function testMembers() { + return [testMember("1"), testMember("2"), testMember("3")]; + } + + function testChannels() { + return [testChannel(activeChannelId), testChannel("2"), testChannel("3")]; + } + + function testMessages(channelId) { + if (channelId === activeChannelId) { + const authorId = "1"; + const reactionEmoji = [customEmoji()]; + return [ + testMessage(activeMessageid, channelId, authorId, reactionEmoji), + testMessage("2", channelId, authorId, []), + ]; + } else { + return []; + } + } + + function testReactions(channelId, messageId) { + if (channelId === activeChannelId && messageId === activeMessageid) { + return [ + testReaction(channelId, messageId, "1"), + testReaction(channelId, messageId, "2"), + ]; + } else { + return []; + } + } + + function mockStream(): DepaginatedFetcher { + return { + members(): Promise<$ReadOnlyArray> { + return Promise.resolve(testMembers()); + }, + channels(): Promise<$ReadOnlyArray> { + return Promise.resolve(testChannels()); + }, + messages(channel: Snowflake): Promise<$ReadOnlyArray> { + return Promise.resolve(testMessages(channel)); + }, + reactions( + channel: Snowflake, + message: Snowflake + ): Promise<$ReadOnlyArray> { + return Promise.resolve(testReactions(channel, message)); + }, + }; + } + + function setupTestData() { + const mirror = testMirror(); + const stream = mockStream(); + return {mirror, stream}; + } + + it("fetches members", async () => { + const {mirror, stream} = setupTestData(); + await fetchDiscord(mirror, stream); + expect(mirror.members()).toEqual(testMembers()); + }); + it("fetches channels", async () => { + const {mirror, stream} = setupTestData(); + await fetchDiscord(mirror, stream); + expect(mirror.channels()).toEqual(testChannels()); + }); + it("fetches messages", async () => { + const {mirror, stream} = setupTestData(); + await fetchDiscord(mirror, stream); + + expect(mirror.messages(activeMessageid)).toEqual(testMessages("1")); + }); + it("fetches reactions", async () => { + const {mirror, stream} = setupTestData(); + await fetchDiscord(mirror, stream); + expect(mirror.reactions(activeMessageid, activeChannelId)).toEqual( + testReactions("1", "1") + ); + }); + }); + + describe("get pages", () => { + it("handles pagination correctly", async () => { + const fetch = jest + .fn() + .mockImplementationOnce(() => + Promise.resolve({ + pageInfo: {hasNextPage: true, endCursor: "1"}, + results: [0, 1], + }) + ) + .mockImplementationOnce(() => + Promise.resolve({ + pageInfo: {hasNextPage: true, endCursor: "3"}, + results: [2, 3], + }) + ) + .mockImplementationOnce(() => + Promise.resolve({ + pageInfo: {hasNextPage: false, endCursor: "4"}, + results: [4], + }) + ); + + const results = await getPages(fetch, "0"); + expect(results).toEqual([0, 1, 2, 3, 4]); + expect(fetch.mock.calls).toEqual([["0"], ["1"], ["3"]]); + }); + }); +}); diff --git a/src/plugins/discord/sqliteMirror.test.js b/src/plugins/discord/sqliteMirror.test.js index 368a332..7d2f029 100644 --- a/src/plugins/discord/sqliteMirror.test.js +++ b/src/plugins/discord/sqliteMirror.test.js @@ -8,47 +8,18 @@ import { type Message, type GuildMember, type Snowflake, - type Emoji, type User, } from "./models"; +import { + testChannel, + testUser, + testMember, + testMessage, + customEmoji, + genericEmoji, +} from "./testUtils"; describe("plugins/discord/sqliteMirror", () => { - const customEmoji = (): Emoji => ({id: "id", name: "name"}); - const genericEmoji = (): Emoji => ({id: null, name: "🐙"}); - - const testChannel = (id: Snowflake): Channel => ({ - id: id, - name: "testChannelName", - type: "GUILD_TEXT", - }); - - const testMessage = ( - id: Snowflake, - channelId: Snowflake, - authorId: Snowflake - ): Message => ({ - id: id, - channelId: channelId, - authorId: authorId, - timestampMs: Date.parse("2020-03-03T23:35:10.615000+00:00"), - content: "Just going to drop this here", - reactionEmoji: [customEmoji()], - nonUserAuthor: false, - mentions: ["1", "23"], - }); - - const testUser = (id: Snowflake): User => ({ - id: id, - username: "username", - discriminator: "disc", - bot: true, - }); - - const testMember = (userId: Snowflake): GuildMember => ({ - user: testUser(userId), - nick: "nickname", - }); - describe("constructor", () => { it("initializes a new database succsessfully", () => { const db = new Database(":memory:"); @@ -261,8 +232,12 @@ describe("plugins/discord/sqliteMirror", () => { const messageId2 = "2"; const channelId = "3"; const authorId = "4"; - const mes1: Message = testMessage(messageId1, channelId, authorId); - const mes2: Message = testMessage(messageId2, channelId, authorId); + const mes1: Message = testMessage(messageId1, channelId, authorId, [ + customEmoji(), + ]); + const mes2: Message = testMessage(messageId2, channelId, authorId, [ + customEmoji(), + ]); const db = new Database(":memory:"); const sqliteMirror = new SqliteMirror(db, "0"); @@ -271,16 +246,6 @@ describe("plugins/discord/sqliteMirror", () => { sqliteMirror.addMessage(mes1); sqliteMirror.addMessage(mes2); - for (const user of mes1.mentions) { - sqliteMirror.addUser(testUser(user)); - sqliteMirror.addMention(mes1, user); - } - - for (const user of mes2.mentions) { - sqliteMirror.addUser(testUser(user)); - sqliteMirror.addMention(mes2, user); - } - for (const emoji of mes1.reactionEmoji) { sqliteMirror.addReaction({ emoji, @@ -407,8 +372,11 @@ describe("plugins/discord/sqliteMirror", () => { const messageId: Snowflake = "1"; const channelId = "2"; const authorId = "3"; - const message: Message = testMessage(messageId, channelId, authorId); - const [userId1, userId2] = message.mentions; + const userId1 = "4"; + const userId2 = "5"; + const message: Message = testMessage(messageId, channelId, authorId, [ + customEmoji(), + ]); const db = new Database(":memory:"); const sqliteMirror = new SqliteMirror(db, "0"); sqliteMirror.addUser(testUser(userId1)); diff --git a/src/plugins/discord/testUtils.js b/src/plugins/discord/testUtils.js new file mode 100644 index 0000000..930c326 --- /dev/null +++ b/src/plugins/discord/testUtils.js @@ -0,0 +1,62 @@ +// @flow + +import { + type Channel, + type Message, + type GuildMember, + type Snowflake, + type Emoji, + type User, + type Reaction, +} from "./models"; + +export const customEmoji = (): Emoji => ({id: "id", name: "name"}); +export const genericEmoji = (): Emoji => ({id: null, name: "🐙"}); + +export const testChannel = (id: Snowflake): Channel => ({ + id: id, + name: "testChannelName", + type: "GUILD_TEXT", +}); + +export const testMessage = ( + id: Snowflake, + channelId: Snowflake, + authorId: Snowflake, + reactionEmoji: ?$ReadOnlyArray, + mentions: ?$ReadOnlyArray +): Message => ({ + id: id, + channelId: channelId, + authorId: authorId, + timestampMs: Date.parse("2020-03-03T23:35:10.615000+00:00"), + content: "Just going to drop this here", + reactionEmoji: reactionEmoji || [], + nonUserAuthor: false, + mentions: mentions || [], +}); + +export const testUser = (id: Snowflake): User => ({ + id: id, + username: "username", + discriminator: "disc", + bot: true, +}); + +export const testMember = (userId: Snowflake): GuildMember => ({ + user: testUser(userId), + nick: "nickname", +}); + +export const testReaction = ( + channelId: Snowflake, + messageId: Snowflake, + authorId: Snowflake +): Reaction => { + return { + channelId: channelId, + messageId: messageId, + authorId: authorId, + emoji: customEmoji(), + }; +};