Initiatives: implement DiscourseInitiativeRepository (#1483)

Uses the Discourse mirror data and parsing function to create an
InitiativeRepository implementation. Which we can use for createGraph.

src/plugins/initiatives/discourse.js

@@ -1,10 +1,106 @@
 // @flow
 
-import type {Topic, Post, TopicId} from "../discourse/fetch";
-import type {Initiative, URL} from "./initiative";
-import type {HtmlTemplateInitiativePartial} from "./htmlTemplate";
+import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch";
+import type {Initiative, URL, InitiativeRepository} from "./initiative";
+import {
+  parseCookedHtml,
+  type HtmlTemplateInitiativePartial,
+} from "./htmlTemplate";
 import {topicAddress} from "../discourse/address";
 
+/**
+ * A subset of queries we need for our plugin.
+ */
+export interface DiscourseQueries {
+  /**
+   * Finds the TopicIds of topics that have one of the categoryIds as it's category.
+   */
+  topicsInCategories(
+    categoryIds: $ReadOnlyArray<CategoryId>
+  ): $ReadOnlyArray<TopicId>;
+
+  /**
+   * Gets a Topic by ID.
+   */
+  topicById(id: TopicId): ?Topic;
+
+  /**
+   * Gets a number of Posts in a given Topic.
+   */
+  postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray<Post>;
+}
+
+type DiscourseInitiativeRepositoryOptions = {|
+  +serverUrl: string,
+  +queries: DiscourseQueries,
+  +initiativesCategory: CategoryId,
+  +topicBlacklist: $ReadOnlyArray<TopicId>,
+  +parseCookedHtml?: (cookedHTML: string) => HtmlTemplateInitiativePartial,
+|};
+
+/**
+ * Repository to get Initiatives from Discourse data.
+ *
+ * Note: will warn about parsing errors and only return Initiatives that could
+ * be parsed successfully.
+ */
+export class DiscourseInitiativeRepository implements InitiativeRepository {
+  _options: DiscourseInitiativeRepositoryOptions;
+
+  constructor(options: DiscourseInitiativeRepositoryOptions) {
+    this._options = options;
+  }
+
+  initiatives(): $ReadOnlyArray<Initiative> {
+    const {
+      serverUrl,
+      queries,
+      initiativesCategory,
+      topicBlacklist,
+    } = this._options;
+    const parser = this._options.parseCookedHtml || parseCookedHtml;
+
+    // Gets a list of TopicIds by category, and remove the blacklisted ones.
+    const topicIds = new Set(queries.topicsInCategories([initiativesCategory]));
+    for (const tid of topicBlacklist) {
+      topicIds.delete(tid);
+    }
+
+    const initiatives = [];
+    const errors = [];
+    const expected = topicIds.size;
+    for (const tid of topicIds) {
+      const topic = queries.topicById(tid);
+      const [openingPost] = queries.postsInTopic(tid, 1);
+
+      if (!topic || !openingPost) {
+        throw new Error("Implementation bug, should have topic and op here.");
+      }
+
+      // We're using parse errors only for informative purposes.
+      // Trap them here and push to errors list.
+      try {
+        initiatives.push(
+          initiativeFromDiscourseTracker(serverUrl, topic, openingPost, parser)
+        );
+      } catch (e) {
+        errors.push(e.message);
+      }
+    }
+
+    // Warn about the issues we've encountered in one go.
+    if (errors.length > 0) {
+      console.warn(
+        `Failed loading [${
+          errors.length
+        }/${expected}] initiatives:\n${errors.join("\n")}`
+      );
+    }
+
+    return initiatives;
+  }
+}
+
 /**
  * Uses data from a Discourse Topic to create an Initiative representation.
  *

src/plugins/initiatives/discourse.test.js

@@ -1,9 +1,15 @@
 // @flow
 
 import {type HtmlTemplateInitiativePartial} from "./htmlTemplate";
-import {initiativeFromDiscourseTracker} from "./discourse";
-import type {Topic, Post} from "../discourse/fetch";
+import {
+  initiativeFromDiscourseTracker,
+  DiscourseInitiativeRepository,
+  type DiscourseQueries,
+} from "./discourse";
+import {type Initiative} from "./initiative";
+import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch";
 import {NodeAddress} from "../../core/graph";
+import dedent from "../../util/dedent";
 
 function givenParseError(message: string) {
   return mockParseCookedHtml(() => {
@@ -21,6 +27,13 @@ function mockParseCookedHtml(
   return jest.fn().mockImplementation(fn);
 }
 
+function snapshotInitiative(initiative: Initiative): Object {
+  return {
+    ...initiative,
+    tracker: NodeAddress.toParts(initiative.tracker),
+  };
+}
+
 function exampleTopic(overrides?: $Shape<Topic>): Topic {
   return {
     id: 123,
@@ -59,9 +72,231 @@ function examplePartialIniative(
   };
 }
 
+function exampleOptions({
+  topics,
+  initiativesCategory,
+  topicBlacklist,
+  parseCookedHtml,
+}: any) {
+  return {
+    serverUrl: "https://foo.bar",
+    topicBlacklist: topicBlacklist || [],
+    initiativesCategory: initiativesCategory || 42,
+    queries: new MockDiscourseQueries(topics || []),
+    parseCookedHtml,
+  };
+}
+
+type TopicWithOpeningPost = {|
+  +topic: Topic,
+  +post: Post,
+|};
+
+class MockDiscourseQueries implements DiscourseQueries {
+  _entries: Map<TopicId, TopicWithOpeningPost>;
+
+  constructor(topics: $Shape<Topic>[]) {
+    this._entries = new Map();
+    jest.spyOn(this, "topicsInCategories");
+    jest.spyOn(this, "topicById");
+    jest.spyOn(this, "postsInTopic");
+
+    let postId = 100;
+    for (const topicShape of topics) {
+      const topic = exampleTopic(topicShape);
+      const post = examplePost({
+        id: postId++,
+        topicId: topic.id,
+      });
+      this._entries.set(topic.id, {topic, post});
+    }
+  }
+
+  topicsInCategories(
+    categoryIds: $ReadOnlyArray<CategoryId>
+  ): $ReadOnlyArray<TopicId> {
+    const ids: TopicId[] = [];
+    for (const {topic} of this._entries.values()) {
+      if (categoryIds.includes(topic.categoryId)) {
+        ids.push(topic.id);
+      }
+    }
+    return ids;
+  }
+
+  topicById(id: TopicId): ?Topic {
+    const pair = this._entries.get(id);
+    return pair ? pair.topic : null;
+  }
+
+  postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray<Post> {
+    if (numberOfPosts != 1) {
+      throw new Error(
+        "MockDiscourseQueries doesn't support anything but 1 for numberOfPosts"
+      );
+    }
+
+    const pair = this._entries.get(topicId);
+    return pair ? [pair.post] : [];
+  }
+}
+
 describe("plugins/initiatives/discourse", () => {
+  function spyWarn(): JestMockFn<[string], void> {
+    return ((console.warn: any): JestMockFn<any, void>);
+  }
   beforeEach(() => {
-    givenParseError("No parseCookedHtml mock value set");
+    jest.spyOn(console, "warn").mockImplementation(() => {});
+  });
+  afterEach(() => {
+    try {
+      expect(console.warn).not.toHaveBeenCalled();
+    } finally {
+      spyWarn().mockRestore();
+    }
+  });
+
+  describe("DiscourseInitiativeRepository", () => {
+    it("uses topicsInCategories to find initiative topics", () => {
+      // Given
+      const options = exampleOptions({
+        initiativesCategory: 16,
+        parseCookedHtml: givenParseResponse(examplePartialIniative()),
+      });
+
+      // When
+      const repo = new DiscourseInitiativeRepository(options);
+      repo.initiatives();
+
+      // Then
+      expect(options.queries.topicsInCategories).toBeCalledTimes(1);
+      expect(options.queries.topicsInCategories).toBeCalledWith([16]);
+    });
+
+    it("attempts to get Topic and opening Post for each TopicId found", () => {
+      // Given
+      const options = exampleOptions({
+        topics: [{id: 40}, {id: 41}, {id: 42}],
+        parseCookedHtml: givenParseResponse(examplePartialIniative()),
+      });
+
+      // When
+      const repo = new DiscourseInitiativeRepository(options);
+      repo.initiatives();
+
+      // Then
+      expect(options.queries.topicById).toBeCalledTimes(3);
+      expect(options.queries.topicById).toBeCalledWith(40);
+      expect(options.queries.topicById).toBeCalledWith(41);
+      expect(options.queries.topicById).toBeCalledWith(42);
+      expect(options.queries.postsInTopic).toBeCalledTimes(3);
+      expect(options.queries.postsInTopic).toBeCalledWith(40, 1);
+      expect(options.queries.postsInTopic).toBeCalledWith(41, 1);
+      expect(options.queries.postsInTopic).toBeCalledWith(42, 1);
+    });
+
+    it("filters blacklisted Topics from TopicId found", () => {
+      // Given
+      const options = exampleOptions({
+        topicBlacklist: [41, 50],
+        topics: [{id: 40}, {id: 41}, {id: 42}],
+        parseCookedHtml: givenParseResponse(examplePartialIniative()),
+      });
+
+      // When
+      const repo = new DiscourseInitiativeRepository(options);
+      repo.initiatives();
+
+      // Then
+      expect(options.queries.topicById).toBeCalledTimes(2);
+      expect(options.queries.topicById).toBeCalledWith(40);
+      expect(options.queries.topicById).toBeCalledWith(42);
+      expect(options.queries.postsInTopic).toBeCalledTimes(2);
+      expect(options.queries.postsInTopic).toBeCalledWith(40, 1);
+      expect(options.queries.postsInTopic).toBeCalledWith(42, 1);
+    });
+
+    it("creates Initiatives for matched Topics", () => {
+      // Given
+      const options = exampleOptions({
+        topics: [{id: 40}, {id: 42}],
+        parseCookedHtml: givenParseResponse(
+          examplePartialIniative({
+            references: ["https://example.org/references/included"],
+          })
+        ),
+      });
+
+      // When
+      const repo = new DiscourseInitiativeRepository(options);
+      const initiatives = repo.initiatives();
+
+      // Then
+      expect(initiatives.map(snapshotInitiative)).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "champions": Array [],
+            "completed": false,
+            "contributions": Array [],
+            "dependencies": Array [],
+            "references": Array [
+              "https://example.org/references/included",
+            ],
+            "timestampMs": 1571498171951,
+            "title": "Example initiative",
+            "tracker": Array [
+              "sourcecred",
+              "discourse",
+              "topic",
+              "https://foo.bar",
+              "40",
+            ],
+          },
+          Object {
+            "champions": Array [],
+            "completed": false,
+            "contributions": Array [],
+            "dependencies": Array [],
+            "references": Array [
+              "https://example.org/references/included",
+            ],
+            "timestampMs": 1571498171951,
+            "title": "Example initiative",
+            "tracker": Array [
+              "sourcecred",
+              "discourse",
+              "topic",
+              "https://foo.bar",
+              "42",
+            ],
+          },
+        ]
+      `);
+    });
+
+    it("warns when Initiatives fail to parse", () => {
+      // Given
+      const options = exampleOptions({
+        topics: [{id: 40}, {id: 42}],
+        parseCookedHtml: givenParseError("Testing parse error"),
+      });
+
+      // When
+      const repo = new DiscourseInitiativeRepository(options);
+      const initiatives = repo.initiatives();
+
+      // Then
+      expect(initiatives).toEqual([]);
+      expect(console.warn).toHaveBeenCalledWith(
+        dedent`
+        Failed loading [2/2] initiatives:
+        Testing parse error for initiative topic "Example initiative" https://foo.bar/t/40
+        Testing parse error for initiative topic "Example initiative" https://foo.bar/t/42
+      `.trim()
+      );
+      expect(console.warn).toHaveBeenCalledTimes(1);
+      spyWarn().mockReset();
+    });
   });
 
   describe("initiativeFromDiscourseTracker", () => {