From 03d525810d4659a4782fc22bb8d789cc0e45e3db Mon Sep 17 00:00:00 2001 From: Robin van Boven <497556+Beanow@users.noreply.github.com> Date: Tue, 7 Jan 2020 14:43:45 +0100 Subject: [PATCH] Initiatives: implement DiscourseInitiativeRepository (#1483) Uses the Discourse mirror data and parsing function to create an InitiativeRepository implementation. Which we can use for createGraph. --- src/plugins/initiatives/discourse.js | 102 ++++++++- src/plugins/initiatives/discourse.test.js | 241 +++++++++++++++++++++- 2 files changed, 337 insertions(+), 6 deletions(-) diff --git a/src/plugins/initiatives/discourse.js b/src/plugins/initiatives/discourse.js index d1b356c..6040052 100644 --- a/src/plugins/initiatives/discourse.js +++ b/src/plugins/initiatives/discourse.js @@ -1,10 +1,106 @@ // @flow -import type {Topic, Post, TopicId} from "../discourse/fetch"; -import type {Initiative, URL} from "./initiative"; -import type {HtmlTemplateInitiativePartial} from "./htmlTemplate"; +import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch"; +import type {Initiative, URL, InitiativeRepository} from "./initiative"; +import { + parseCookedHtml, + type HtmlTemplateInitiativePartial, +} from "./htmlTemplate"; import {topicAddress} from "../discourse/address"; +/** + * A subset of queries we need for our plugin. + */ +export interface DiscourseQueries { + /** + * Finds the TopicIds of topics that have one of the categoryIds as it's category. + */ + topicsInCategories( + categoryIds: $ReadOnlyArray + ): $ReadOnlyArray; + + /** + * Gets a Topic by ID. + */ + topicById(id: TopicId): ?Topic; + + /** + * Gets a number of Posts in a given Topic. + */ + postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray; +} + +type DiscourseInitiativeRepositoryOptions = {| + +serverUrl: string, + +queries: DiscourseQueries, + +initiativesCategory: CategoryId, + +topicBlacklist: $ReadOnlyArray, + +parseCookedHtml?: (cookedHTML: string) => HtmlTemplateInitiativePartial, +|}; + +/** + * Repository to get Initiatives from Discourse data. + * + * Note: will warn about parsing errors and only return Initiatives that could + * be parsed successfully. + */ +export class DiscourseInitiativeRepository implements InitiativeRepository { + _options: DiscourseInitiativeRepositoryOptions; + + constructor(options: DiscourseInitiativeRepositoryOptions) { + this._options = options; + } + + initiatives(): $ReadOnlyArray { + const { + serverUrl, + queries, + initiativesCategory, + topicBlacklist, + } = this._options; + const parser = this._options.parseCookedHtml || parseCookedHtml; + + // Gets a list of TopicIds by category, and remove the blacklisted ones. + const topicIds = new Set(queries.topicsInCategories([initiativesCategory])); + for (const tid of topicBlacklist) { + topicIds.delete(tid); + } + + const initiatives = []; + const errors = []; + const expected = topicIds.size; + for (const tid of topicIds) { + const topic = queries.topicById(tid); + const [openingPost] = queries.postsInTopic(tid, 1); + + if (!topic || !openingPost) { + throw new Error("Implementation bug, should have topic and op here."); + } + + // We're using parse errors only for informative purposes. + // Trap them here and push to errors list. + try { + initiatives.push( + initiativeFromDiscourseTracker(serverUrl, topic, openingPost, parser) + ); + } catch (e) { + errors.push(e.message); + } + } + + // Warn about the issues we've encountered in one go. + if (errors.length > 0) { + console.warn( + `Failed loading [${ + errors.length + }/${expected}] initiatives:\n${errors.join("\n")}` + ); + } + + return initiatives; + } +} + /** * Uses data from a Discourse Topic to create an Initiative representation. * diff --git a/src/plugins/initiatives/discourse.test.js b/src/plugins/initiatives/discourse.test.js index 2167f33..0ea8a44 100644 --- a/src/plugins/initiatives/discourse.test.js +++ b/src/plugins/initiatives/discourse.test.js @@ -1,9 +1,15 @@ // @flow import {type HtmlTemplateInitiativePartial} from "./htmlTemplate"; -import {initiativeFromDiscourseTracker} from "./discourse"; -import type {Topic, Post} from "../discourse/fetch"; +import { + initiativeFromDiscourseTracker, + DiscourseInitiativeRepository, + type DiscourseQueries, +} from "./discourse"; +import {type Initiative} from "./initiative"; +import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch"; import {NodeAddress} from "../../core/graph"; +import dedent from "../../util/dedent"; function givenParseError(message: string) { return mockParseCookedHtml(() => { @@ -21,6 +27,13 @@ function mockParseCookedHtml( return jest.fn().mockImplementation(fn); } +function snapshotInitiative(initiative: Initiative): Object { + return { + ...initiative, + tracker: NodeAddress.toParts(initiative.tracker), + }; +} + function exampleTopic(overrides?: $Shape): Topic { return { id: 123, @@ -59,9 +72,231 @@ function examplePartialIniative( }; } +function exampleOptions({ + topics, + initiativesCategory, + topicBlacklist, + parseCookedHtml, +}: any) { + return { + serverUrl: "https://foo.bar", + topicBlacklist: topicBlacklist || [], + initiativesCategory: initiativesCategory || 42, + queries: new MockDiscourseQueries(topics || []), + parseCookedHtml, + }; +} + +type TopicWithOpeningPost = {| + +topic: Topic, + +post: Post, +|}; + +class MockDiscourseQueries implements DiscourseQueries { + _entries: Map; + + constructor(topics: $Shape[]) { + this._entries = new Map(); + jest.spyOn(this, "topicsInCategories"); + jest.spyOn(this, "topicById"); + jest.spyOn(this, "postsInTopic"); + + let postId = 100; + for (const topicShape of topics) { + const topic = exampleTopic(topicShape); + const post = examplePost({ + id: postId++, + topicId: topic.id, + }); + this._entries.set(topic.id, {topic, post}); + } + } + + topicsInCategories( + categoryIds: $ReadOnlyArray + ): $ReadOnlyArray { + const ids: TopicId[] = []; + for (const {topic} of this._entries.values()) { + if (categoryIds.includes(topic.categoryId)) { + ids.push(topic.id); + } + } + return ids; + } + + topicById(id: TopicId): ?Topic { + const pair = this._entries.get(id); + return pair ? pair.topic : null; + } + + postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray { + if (numberOfPosts != 1) { + throw new Error( + "MockDiscourseQueries doesn't support anything but 1 for numberOfPosts" + ); + } + + const pair = this._entries.get(topicId); + return pair ? [pair.post] : []; + } +} + describe("plugins/initiatives/discourse", () => { + function spyWarn(): JestMockFn<[string], void> { + return ((console.warn: any): JestMockFn); + } beforeEach(() => { - givenParseError("No parseCookedHtml mock value set"); + jest.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + try { + expect(console.warn).not.toHaveBeenCalled(); + } finally { + spyWarn().mockRestore(); + } + }); + + describe("DiscourseInitiativeRepository", () => { + it("uses topicsInCategories to find initiative topics", () => { + // Given + const options = exampleOptions({ + initiativesCategory: 16, + parseCookedHtml: givenParseResponse(examplePartialIniative()), + }); + + // When + const repo = new DiscourseInitiativeRepository(options); + repo.initiatives(); + + // Then + expect(options.queries.topicsInCategories).toBeCalledTimes(1); + expect(options.queries.topicsInCategories).toBeCalledWith([16]); + }); + + it("attempts to get Topic and opening Post for each TopicId found", () => { + // Given + const options = exampleOptions({ + topics: [{id: 40}, {id: 41}, {id: 42}], + parseCookedHtml: givenParseResponse(examplePartialIniative()), + }); + + // When + const repo = new DiscourseInitiativeRepository(options); + repo.initiatives(); + + // Then + expect(options.queries.topicById).toBeCalledTimes(3); + expect(options.queries.topicById).toBeCalledWith(40); + expect(options.queries.topicById).toBeCalledWith(41); + expect(options.queries.topicById).toBeCalledWith(42); + expect(options.queries.postsInTopic).toBeCalledTimes(3); + expect(options.queries.postsInTopic).toBeCalledWith(40, 1); + expect(options.queries.postsInTopic).toBeCalledWith(41, 1); + expect(options.queries.postsInTopic).toBeCalledWith(42, 1); + }); + + it("filters blacklisted Topics from TopicId found", () => { + // Given + const options = exampleOptions({ + topicBlacklist: [41, 50], + topics: [{id: 40}, {id: 41}, {id: 42}], + parseCookedHtml: givenParseResponse(examplePartialIniative()), + }); + + // When + const repo = new DiscourseInitiativeRepository(options); + repo.initiatives(); + + // Then + expect(options.queries.topicById).toBeCalledTimes(2); + expect(options.queries.topicById).toBeCalledWith(40); + expect(options.queries.topicById).toBeCalledWith(42); + expect(options.queries.postsInTopic).toBeCalledTimes(2); + expect(options.queries.postsInTopic).toBeCalledWith(40, 1); + expect(options.queries.postsInTopic).toBeCalledWith(42, 1); + }); + + it("creates Initiatives for matched Topics", () => { + // Given + const options = exampleOptions({ + topics: [{id: 40}, {id: 42}], + parseCookedHtml: givenParseResponse( + examplePartialIniative({ + references: ["https://example.org/references/included"], + }) + ), + }); + + // When + const repo = new DiscourseInitiativeRepository(options); + const initiatives = repo.initiatives(); + + // Then + expect(initiatives.map(snapshotInitiative)).toMatchInlineSnapshot(` + Array [ + Object { + "champions": Array [], + "completed": false, + "contributions": Array [], + "dependencies": Array [], + "references": Array [ + "https://example.org/references/included", + ], + "timestampMs": 1571498171951, + "title": "Example initiative", + "tracker": Array [ + "sourcecred", + "discourse", + "topic", + "https://foo.bar", + "40", + ], + }, + Object { + "champions": Array [], + "completed": false, + "contributions": Array [], + "dependencies": Array [], + "references": Array [ + "https://example.org/references/included", + ], + "timestampMs": 1571498171951, + "title": "Example initiative", + "tracker": Array [ + "sourcecred", + "discourse", + "topic", + "https://foo.bar", + "42", + ], + }, + ] + `); + }); + + it("warns when Initiatives fail to parse", () => { + // Given + const options = exampleOptions({ + topics: [{id: 40}, {id: 42}], + parseCookedHtml: givenParseError("Testing parse error"), + }); + + // When + const repo = new DiscourseInitiativeRepository(options); + const initiatives = repo.initiatives(); + + // Then + expect(initiatives).toEqual([]); + expect(console.warn).toHaveBeenCalledWith( + dedent` + Failed loading [2/2] initiatives: + Testing parse error for initiative topic "Example initiative" https://foo.bar/t/40 + Testing parse error for initiative topic "Example initiative" https://foo.bar/t/42 + `.trim() + ); + expect(console.warn).toHaveBeenCalledTimes(1); + spyWarn().mockReset(); + }); }); describe("initiativeFromDiscourseTracker", () => {