Discord: add Mirror class (#1780)

Creates a class to handle the implementation
of fetching data from the Discord server and
saving it to the local mirror.

Test plan:

Unit tests added.

paired with @beanow

src/plugins/discord/fetcher.js

@@ -25,6 +25,30 @@ export type ResultPage<T> = {|
   +results: $ReadOnlyArray<T>,
 |};
 
+/**
+ * An interface to fetch Discord data
+ */
+export interface DiscordFetcher {
+  channels(guildId: Snowflake): Promise<$ReadOnlyArray<Model.Channel>>;
+
+  members(
+    guildId: Snowflake,
+    after: Snowflake
+  ): Promise<ResultPage<Model.GuildMember>>;
+
+  messages(
+    channel: Snowflake,
+    after: Snowflake
+  ): Promise<ResultPage<Model.Message>>;
+
+  reactions(
+    channel: Snowflake,
+    message: Snowflake,
+    emoji: Model.Emoji,
+    after: Snowflake
+  ): Promise<ResultPage<Model.Reaction>>;
+}
+
 /**
  * Fetcher is responsible for:
  * - Returning the correct endpoint to fetch against for Guilds, Channels,
@@ -43,7 +67,7 @@ export type ResultPage<T> = {|
  *   returning an array of Channel objects in the corresponding method.
  *   See: https://discordapp.com/developers/docs/resources/guild#get-guild-channels
  */
-export class Fetcher {
+export class Fetcher implements DiscordFetcher {
   +_fetch: FetchEndpoint;
   +_options: FetchOptions;
 

src/plugins/discord/fetcher.test.js

@@ -7,9 +7,9 @@ import {
   type Reaction,
   type Message,
   type GuildMember,
-  type Snowflake,
   type Emoji,
 } from "./models";
+import {testChannel} from "./testUtils";
 
 /**
  * Note: the 'any' type signature is assigned to server response objects
@@ -50,20 +50,14 @@ describe("plugins/discord/fetcher", () => {
     });
 
     it("handles response", async () => {
-      const testChannel = (id: string): any => ({
+      const testChannelResp = (id: string): any => ({
         id: id,
         name: "testChannelName",
         type: 0,
       });
 
-      const testChannelObj = (id: Snowflake): Channel => ({
+      const response: any[] = [testChannelResp("1")];
-        id: id,
+      const expected: Channel[] = [testChannel("1")];
-        name: "testChannelName",
-        type: "GUILD_TEXT",
-      });
-
-      const response: any[] = [testChannel("1")];
-      const expected: Channel[] = [testChannelObj("1")];
       const fetch = jest.fn(() => Promise.resolve(response));
       const fetcher = new Fetcher(fetch, defaultOptions());
       const data = await fetcher.channels("0");

src/plugins/discord/mirror.js

@@ -0,0 +1,78 @@
+// @flow
+
+import {type ResultPage} from "./fetcher";
+import {type SqliteMirror} from "./sqliteMirror";
+import {type Snowflake} from "./models";
+import * as Model from "./models";
+import * as nullUtil from "../../util/null";
+
+/**
+ * Mirrors data from the Discord API into a local sqlite db.
+ */
+
+export async function fetchDiscord(
+  sqliteMirror: SqliteMirror,
+  streams: DepaginatedFetcher
+) {
+  for (const member of await streams.members()) {
+    sqliteMirror.addMember(member);
+  }
+
+  for (const channel of await streams.channels()) {
+    sqliteMirror.addChannel(channel);
+
+    for (const message of await streams.messages(channel.id)) {
+      sqliteMirror.addMessage(message);
+
+      for (const emoji of message.reactionEmoji) {
+        for (const reaction of await streams.reactions(
+          channel.id,
+          message.id,
+          emoji
+        )) {
+          sqliteMirror.addReaction(reaction);
+        }
+      }
+    }
+  }
+}
+
+// Note: most of this is about wrapping pagination.
+export interface DepaginatedFetcher {
+  members(after: ?Snowflake): Promise<$ReadOnlyArray<Model.GuildMember>>;
+  channels(): Promise<$ReadOnlyArray<Model.Channel>>;
+  messages(
+    channel: Snowflake,
+    after: ?Snowflake
+  ): Promise<$ReadOnlyArray<Model.Message>>;
+  reactions(
+    channel: Snowflake,
+    message: Snowflake,
+    emoji: Model.Emoji,
+    after: ?Snowflake
+  ): Promise<$ReadOnlyArray<Model.Reaction>>;
+}
+
+export async function getPages<T>(
+  fetchPage: (endCursor: Snowflake) => Promise<ResultPage<T>>,
+  endCursor: Snowflake
+): Promise<$ReadOnlyArray<T>> {
+  let after = endCursor;
+  let hasNextPage = true;
+  let allResults = [];
+
+  while (hasNextPage) {
+    const {results, pageInfo} = await fetchPage(after);
+    allResults = allResults.concat(results);
+
+    hasNextPage = pageInfo.hasNextPage;
+    if (hasNextPage) {
+      after = nullUtil.get(
+        pageInfo.endCursor,
+        `Found null endCursor with hasNextPage = true`
+      );
+    }
+  }
+
+  return allResults;
+}

src/plugins/discord/mirror.test.js

@@ -0,0 +1,137 @@
+// @flow
+
+import Database from "better-sqlite3";
+import {fetchDiscord, DepaginatedFetcher, getPages} from "./mirror";
+import {SqliteMirror} from "./sqliteMirror";
+import * as Model from "./models";
+import {type Snowflake} from "./models";
+import {
+  testChannel,
+  testMember,
+  testMessage,
+  customEmoji,
+  testReaction,
+} from "./testUtils";
+
+describe("plugins/discord/mirror", () => {
+  describe("updateMirror", () => {
+    const activeChannelId = "12";
+    const activeMessageid = "34";
+
+    function testMirror() {
+      const db = new Database(":memory:");
+      return new SqliteMirror(db, "0");
+    }
+
+    function testMembers() {
+      return [testMember("1"), testMember("2"), testMember("3")];
+    }
+
+    function testChannels() {
+      return [testChannel(activeChannelId), testChannel("2"), testChannel("3")];
+    }
+
+    function testMessages(channelId) {
+      if (channelId === activeChannelId) {
+        const authorId = "1";
+        const reactionEmoji = [customEmoji()];
+        return [
+          testMessage(activeMessageid, channelId, authorId, reactionEmoji),
+          testMessage("2", channelId, authorId, []),
+        ];
+      } else {
+        return [];
+      }
+    }
+
+    function testReactions(channelId, messageId) {
+      if (channelId === activeChannelId && messageId === activeMessageid) {
+        return [
+          testReaction(channelId, messageId, "1"),
+          testReaction(channelId, messageId, "2"),
+        ];
+      } else {
+        return [];
+      }
+    }
+
+    function mockStream(): DepaginatedFetcher {
+      return {
+        members(): Promise<$ReadOnlyArray<Model.GuildMember>> {
+          return Promise.resolve(testMembers());
+        },
+        channels(): Promise<$ReadOnlyArray<Model.Channel>> {
+          return Promise.resolve(testChannels());
+        },
+        messages(channel: Snowflake): Promise<$ReadOnlyArray<Model.Message>> {
+          return Promise.resolve(testMessages(channel));
+        },
+        reactions(
+          channel: Snowflake,
+          message: Snowflake
+        ): Promise<$ReadOnlyArray<Model.Reaction>> {
+          return Promise.resolve(testReactions(channel, message));
+        },
+      };
+    }
+
+    function setupTestData() {
+      const mirror = testMirror();
+      const stream = mockStream();
+      return {mirror, stream};
+    }
+
+    it("fetches members", async () => {
+      const {mirror, stream} = setupTestData();
+      await fetchDiscord(mirror, stream);
+      expect(mirror.members()).toEqual(testMembers());
+    });
+    it("fetches channels", async () => {
+      const {mirror, stream} = setupTestData();
+      await fetchDiscord(mirror, stream);
+      expect(mirror.channels()).toEqual(testChannels());
+    });
+    it("fetches messages", async () => {
+      const {mirror, stream} = setupTestData();
+      await fetchDiscord(mirror, stream);
+
+      expect(mirror.messages(activeMessageid)).toEqual(testMessages("1"));
+    });
+    it("fetches reactions", async () => {
+      const {mirror, stream} = setupTestData();
+      await fetchDiscord(mirror, stream);
+      expect(mirror.reactions(activeMessageid, activeChannelId)).toEqual(
+        testReactions("1", "1")
+      );
+    });
+  });
+
+  describe("get pages", () => {
+    it("handles pagination correctly", async () => {
+      const fetch = jest
+        .fn()
+        .mockImplementationOnce(() =>
+          Promise.resolve({
+            pageInfo: {hasNextPage: true, endCursor: "1"},
+            results: [0, 1],
+          })
+        )
+        .mockImplementationOnce(() =>
+          Promise.resolve({
+            pageInfo: {hasNextPage: true, endCursor: "3"},
+            results: [2, 3],
+          })
+        )
+        .mockImplementationOnce(() =>
+          Promise.resolve({
+            pageInfo: {hasNextPage: false, endCursor: "4"},
+            results: [4],
+          })
+        );
+
+      const results = await getPages(fetch, "0");
+      expect(results).toEqual([0, 1, 2, 3, 4]);
+      expect(fetch.mock.calls).toEqual([["0"], ["1"], ["3"]]);
+    });
+  });
+});

src/plugins/discord/sqliteMirror.test.js

@@ -8,47 +8,18 @@ import {
   type Message,
   type GuildMember,
   type Snowflake,
-  type Emoji,
   type User,
 } from "./models";
+import {
+  testChannel,
+  testUser,
+  testMember,
+  testMessage,
+  customEmoji,
+  genericEmoji,
+} from "./testUtils";
 
 describe("plugins/discord/sqliteMirror", () => {
-  const customEmoji = (): Emoji => ({id: "id", name: "name"});
-  const genericEmoji = (): Emoji => ({id: null, name: "🐙"});
-
-  const testChannel = (id: Snowflake): Channel => ({
-    id: id,
-    name: "testChannelName",
-    type: "GUILD_TEXT",
-  });
-
-  const testMessage = (
-    id: Snowflake,
-    channelId: Snowflake,
-    authorId: Snowflake
-  ): Message => ({
-    id: id,
-    channelId: channelId,
-    authorId: authorId,
-    timestampMs: Date.parse("2020-03-03T23:35:10.615000+00:00"),
-    content: "Just going to drop this here",
-    reactionEmoji: [customEmoji()],
-    nonUserAuthor: false,
-    mentions: ["1", "23"],
-  });
-
-  const testUser = (id: Snowflake): User => ({
-    id: id,
-    username: "username",
-    discriminator: "disc",
-    bot: true,
-  });
-
-  const testMember = (userId: Snowflake): GuildMember => ({
-    user: testUser(userId),
-    nick: "nickname",
-  });
-
   describe("constructor", () => {
     it("initializes a new database succsessfully", () => {
       const db = new Database(":memory:");
@@ -261,8 +232,12 @@ describe("plugins/discord/sqliteMirror", () => {
       const messageId2 = "2";
       const channelId = "3";
       const authorId = "4";
-      const mes1: Message = testMessage(messageId1, channelId, authorId);
+      const mes1: Message = testMessage(messageId1, channelId, authorId, [
-      const mes2: Message = testMessage(messageId2, channelId, authorId);
+        customEmoji(),
+      ]);
+      const mes2: Message = testMessage(messageId2, channelId, authorId, [
+        customEmoji(),
+      ]);
 
       const db = new Database(":memory:");
       const sqliteMirror = new SqliteMirror(db, "0");
@@ -271,16 +246,6 @@ describe("plugins/discord/sqliteMirror", () => {
       sqliteMirror.addMessage(mes1);
       sqliteMirror.addMessage(mes2);
 
-      for (const user of mes1.mentions) {
-        sqliteMirror.addUser(testUser(user));
-        sqliteMirror.addMention(mes1, user);
-      }
-
-      for (const user of mes2.mentions) {
-        sqliteMirror.addUser(testUser(user));
-        sqliteMirror.addMention(mes2, user);
-      }
-
       for (const emoji of mes1.reactionEmoji) {
         sqliteMirror.addReaction({
           emoji,
@@ -407,8 +372,11 @@ describe("plugins/discord/sqliteMirror", () => {
       const messageId: Snowflake = "1";
       const channelId = "2";
       const authorId = "3";
-      const message: Message = testMessage(messageId, channelId, authorId);
+      const userId1 = "4";
-      const [userId1, userId2] = message.mentions;
+      const userId2 = "5";
+      const message: Message = testMessage(messageId, channelId, authorId, [
+        customEmoji(),
+      ]);
       const db = new Database(":memory:");
       const sqliteMirror = new SqliteMirror(db, "0");
       sqliteMirror.addUser(testUser(userId1));

src/plugins/discord/testUtils.js

@@ -0,0 +1,62 @@
+// @flow
+
+import {
+  type Channel,
+  type Message,
+  type GuildMember,
+  type Snowflake,
+  type Emoji,
+  type User,
+  type Reaction,
+} from "./models";
+
+export const customEmoji = (): Emoji => ({id: "id", name: "name"});
+export const genericEmoji = (): Emoji => ({id: null, name: "🐙"});
+
+export const testChannel = (id: Snowflake): Channel => ({
+  id: id,
+  name: "testChannelName",
+  type: "GUILD_TEXT",
+});
+
+export const testMessage = (
+  id: Snowflake,
+  channelId: Snowflake,
+  authorId: Snowflake,
+  reactionEmoji: ?$ReadOnlyArray<Emoji>,
+  mentions: ?$ReadOnlyArray<Snowflake>
+): Message => ({
+  id: id,
+  channelId: channelId,
+  authorId: authorId,
+  timestampMs: Date.parse("2020-03-03T23:35:10.615000+00:00"),
+  content: "Just going to drop this here",
+  reactionEmoji: reactionEmoji || [],
+  nonUserAuthor: false,
+  mentions: mentions || [],
+});
+
+export const testUser = (id: Snowflake): User => ({
+  id: id,
+  username: "username",
+  discriminator: "disc",
+  bot: true,
+});
+
+export const testMember = (userId: Snowflake): GuildMember => ({
+  user: testUser(userId),
+  nick: "nickname",
+});
+
+export const testReaction = (
+  channelId: Snowflake,
+  messageId: Snowflake,
+  authorId: Snowflake
+): Reaction => {
+  return {
+    channelId: channelId,
+    messageId: messageId,
+    authorId: authorId,
+    emoji: customEmoji(),
+  };
+};