Initiatives: remove Discourse related features (#1744)
Previously we intended to use Discourse as a source of Initiatives. Since we're not taking this approach anymore, we're removing the related features here.
This commit is contained in:
Robin van Boven
GitHub
parent
d83cbd98bf
commit
1b3d1b48a1
|
|
@ -1,182 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch";
|
||||
import type {Initiative, URL, InitiativeRepository} from "./initiative";
|
||||
import {createId} from "./initiative";
|
||||
import {
|
||||
parseCookedHtml,
|
||||
type HtmlTemplateInitiativePartial,
|
||||
} from "./htmlTemplate";
|
||||
|
||||
export const DISCOURSE_TOPIC_SUBTYPE = "DISCOURSE_TOPIC";
|
||||
|
||||
/**
|
||||
* A subset of queries we need for our plugin.
|
||||
*/
|
||||
export interface DiscourseQueries {
|
||||
/**
|
||||
* Finds the TopicIds of topics that have one of the categoryIds as it's category.
|
||||
*/
|
||||
topicsInCategories(
|
||||
categoryIds: $ReadOnlyArray<CategoryId>
|
||||
): $ReadOnlyArray<TopicId>;
|
||||
|
||||
/**
|
||||
* Gets a Topic by ID.
|
||||
*/
|
||||
topicById(id: TopicId): ?Topic;
|
||||
|
||||
/**
|
||||
* Gets a number of Posts in a given Topic.
|
||||
*/
|
||||
postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray<Post>;
|
||||
}
|
||||
|
||||
type DiscourseInitiativeRepositoryOptions = {|
|
||||
+serverUrl: string,
|
||||
+queries: DiscourseQueries,
|
||||
+initiativesCategory: CategoryId,
|
||||
+topicBlacklist: $ReadOnlyArray<TopicId>,
|
||||
+parseCookedHtml?: (cookedHTML: string) => HtmlTemplateInitiativePartial,
|
||||
|};
|
||||
|
||||
/**
|
||||
* Repository to get Initiatives from Discourse data.
|
||||
*
|
||||
* Note: will warn about parsing errors and only return Initiatives that could
|
||||
* be parsed successfully.
|
||||
*/
|
||||
export class DiscourseInitiativeRepository implements InitiativeRepository {
|
||||
_options: DiscourseInitiativeRepositoryOptions;
|
||||
|
||||
constructor(options: DiscourseInitiativeRepositoryOptions) {
|
||||
this._options = options;
|
||||
}
|
||||
|
||||
initiatives(): $ReadOnlyArray<Initiative> {
|
||||
const {
|
||||
serverUrl,
|
||||
queries,
|
||||
initiativesCategory,
|
||||
topicBlacklist,
|
||||
} = this._options;
|
||||
const parser = this._options.parseCookedHtml || parseCookedHtml;
|
||||
|
||||
// Gets a list of TopicIds by category, and remove the blacklisted ones.
|
||||
const topicIds = new Set(queries.topicsInCategories([initiativesCategory]));
|
||||
for (const tid of topicBlacklist) {
|
||||
topicIds.delete(tid);
|
||||
}
|
||||
|
||||
const initiatives = [];
|
||||
const errors = [];
|
||||
const expected = topicIds.size;
|
||||
for (const tid of topicIds) {
|
||||
const topic = queries.topicById(tid);
|
||||
const [openingPost] = queries.postsInTopic(tid, 1);
|
||||
|
||||
if (!topic || !openingPost) {
|
||||
throw new Error("Implementation bug, should have topic and op here.");
|
||||
}
|
||||
|
||||
// We're using parse errors only for informative purposes.
|
||||
// Trap them here and push to errors list.
|
||||
try {
|
||||
initiatives.push(
|
||||
initiativeFromDiscourseTracker(serverUrl, topic, openingPost, parser)
|
||||
);
|
||||
} catch (e) {
|
||||
errors.push(e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Warn about the issues we've encountered in one go.
|
||||
if (errors.length > 0) {
|
||||
console.warn(
|
||||
`Failed loading [${
|
||||
errors.length
|
||||
}/${expected}] initiatives:\n${errors.join("\n")}`
|
||||
);
|
||||
}
|
||||
|
||||
return initiatives;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses data from a Discourse Topic to create an Initiative representation.
|
||||
*
|
||||
* The Post should be the opening post of the Topic.
|
||||
* The Post body should adhere to the `parseCookedHtml` expected template.
|
||||
*/
|
||||
export function initiativeFromDiscourseTracker(
|
||||
serverUrl: string,
|
||||
topic: Topic,
|
||||
openingPost: Post,
|
||||
parseCookedHtml: (cookedHTML: string) => HtmlTemplateInitiativePartial
|
||||
): Initiative {
|
||||
if (serverUrl.endsWith("/")) {
|
||||
throw new Error("serverUrl shouldn't end with trailing slash");
|
||||
}
|
||||
|
||||
const {title} = topic;
|
||||
const {timestampMs} = openingPost;
|
||||
try {
|
||||
if (openingPost.topicId !== topic.id) {
|
||||
throw new Error(`Post ${openingPost.id} is from a different topic`);
|
||||
}
|
||||
|
||||
if (openingPost.indexWithinTopic !== 1) {
|
||||
throw new Error(
|
||||
`Post ${openingPost.id} is not the first post in the topic`
|
||||
);
|
||||
}
|
||||
|
||||
const partial = parseCookedHtml(openingPost.cooked);
|
||||
return {
|
||||
id: createId(DISCOURSE_TOPIC_SUBTYPE, serverUrl, String(topic.id)),
|
||||
title,
|
||||
timestampMs,
|
||||
completed: partial.completed,
|
||||
dependencies: absoluteURLs(serverUrl, partial.dependencies),
|
||||
references: absoluteURLs(serverUrl, partial.references),
|
||||
contributions: absoluteURLs(serverUrl, partial.contributions),
|
||||
champions: absoluteURLs(serverUrl, partial.champions),
|
||||
};
|
||||
} catch (e) {
|
||||
// To make solving issues easier, add which initiative topic caused the problem.
|
||||
e.message = `${e.message} for initiative topic "${title}" ${topicUrl(
|
||||
serverUrl,
|
||||
topic.id
|
||||
)}`;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to create a topic URL.
|
||||
*/
|
||||
function topicUrl(serverUrl: string, topicId: TopicId) {
|
||||
// Note: this format doesn't include the "url-friendly-title" infix.
|
||||
// Favoring simplicity, this URL will redirect to include it while being valid.
|
||||
return `${serverUrl}/t/${topicId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes a best effort absolute URL.
|
||||
*
|
||||
* Only supports prefixing the serverUrl when the URL starts with a "/".
|
||||
* Other cases should fail later on, such as for reference detection.
|
||||
*/
|
||||
function absoluteURLs(
|
||||
serverUrl: string,
|
||||
urls: $ReadOnlyArray<URL>
|
||||
): $ReadOnlyArray<URL> {
|
||||
return urls.map((url) => {
|
||||
if (url.startsWith("/")) {
|
||||
return `${serverUrl}${url}`;
|
||||
}
|
||||
|
||||
return url;
|
||||
});
|
||||
}
|
||||
|
|
@ -1,499 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import {type HtmlTemplateInitiativePartial} from "./htmlTemplate";
|
||||
import {
|
||||
initiativeFromDiscourseTracker,
|
||||
DiscourseInitiativeRepository,
|
||||
type DiscourseQueries,
|
||||
} from "./discourse";
|
||||
import type {ReadRepository} from "../discourse/mirrorRepository";
|
||||
import type {Topic, Post, CategoryId, TopicId} from "../discourse/fetch";
|
||||
import dedent from "../../util/dedent";
|
||||
|
||||
function givenParseError(message: string) {
|
||||
return mockParseCookedHtml(() => {
|
||||
throw new Error(message);
|
||||
});
|
||||
}
|
||||
|
||||
function givenParseResponse(value: HtmlTemplateInitiativePartial) {
|
||||
return mockParseCookedHtml(() => ({...value}));
|
||||
}
|
||||
|
||||
function mockParseCookedHtml(
|
||||
fn: () => HtmlTemplateInitiativePartial
|
||||
): (cookedHTML: string) => HtmlTemplateInitiativePartial {
|
||||
return jest.fn().mockImplementation(fn);
|
||||
}
|
||||
|
||||
function exampleTopic(overrides?: $Shape<Topic>): Topic {
|
||||
return {
|
||||
id: 123,
|
||||
categoryId: 42,
|
||||
title: "Example initiative",
|
||||
timestampMs: 1571498171951,
|
||||
bumpedMs: 1571498171951,
|
||||
authorUsername: "TestUser",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function examplePost(overrides?: $Shape<Post>): Post {
|
||||
return {
|
||||
id: 432,
|
||||
topicId: 123,
|
||||
indexWithinTopic: 1,
|
||||
replyToPostIndex: null,
|
||||
timestampMs: 1571498171951,
|
||||
authorUsername: "TestUser",
|
||||
cooked: "",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function examplePartialIniative(
|
||||
overrides?: $Shape<HtmlTemplateInitiativePartial>
|
||||
): HtmlTemplateInitiativePartial {
|
||||
return {
|
||||
completed: false,
|
||||
champions: [],
|
||||
dependencies: [],
|
||||
references: [],
|
||||
contributions: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function exampleOptions({
|
||||
topics,
|
||||
initiativesCategory,
|
||||
topicBlacklist,
|
||||
parseCookedHtml,
|
||||
}: any) {
|
||||
return {
|
||||
serverUrl: "https://foo.bar",
|
||||
topicBlacklist: topicBlacklist || [],
|
||||
initiativesCategory: initiativesCategory || 42,
|
||||
queries: new MockDiscourseQueries(topics || []),
|
||||
parseCookedHtml,
|
||||
};
|
||||
}
|
||||
|
||||
type TopicWithOpeningPost = {|
|
||||
+topic: Topic,
|
||||
+post: Post,
|
||||
|};
|
||||
|
||||
class MockDiscourseQueries implements DiscourseQueries {
|
||||
_entries: Map<TopicId, TopicWithOpeningPost>;
|
||||
|
||||
constructor(topics: $Shape<Topic>[]) {
|
||||
this._entries = new Map();
|
||||
jest.spyOn(this, "topicsInCategories");
|
||||
jest.spyOn(this, "topicById");
|
||||
jest.spyOn(this, "postsInTopic");
|
||||
|
||||
let postId = 100;
|
||||
for (const topicShape of topics) {
|
||||
const topic = exampleTopic(topicShape);
|
||||
const post = examplePost({
|
||||
id: postId++,
|
||||
topicId: topic.id,
|
||||
});
|
||||
this._entries.set(topic.id, {topic, post});
|
||||
}
|
||||
}
|
||||
|
||||
topicsInCategories(
|
||||
categoryIds: $ReadOnlyArray<CategoryId>
|
||||
): $ReadOnlyArray<TopicId> {
|
||||
const ids: TopicId[] = [];
|
||||
for (const {topic} of this._entries.values()) {
|
||||
if (categoryIds.includes(topic.categoryId)) {
|
||||
ids.push(topic.id);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
topicById(id: TopicId): ?Topic {
|
||||
const pair = this._entries.get(id);
|
||||
return pair ? pair.topic : null;
|
||||
}
|
||||
|
||||
postsInTopic(topicId: TopicId, numberOfPosts: number): $ReadOnlyArray<Post> {
|
||||
if (numberOfPosts !== 1) {
|
||||
throw new Error(
|
||||
"MockDiscourseQueries doesn't support anything but 1 for numberOfPosts"
|
||||
);
|
||||
}
|
||||
|
||||
const pair = this._entries.get(topicId);
|
||||
return pair ? [pair.post] : [];
|
||||
}
|
||||
}
|
||||
|
||||
describe("plugins/initiatives/discourse", () => {
|
||||
function spyWarn(): JestMockFn<[string], void> {
|
||||
return ((console.warn: any): JestMockFn<any, void>);
|
||||
}
|
||||
beforeEach(() => {
|
||||
jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(() => {
|
||||
try {
|
||||
expect(console.warn).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
spyWarn().mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
describe("DiscourseQueries", () => {
|
||||
it("should be a subset of the ReadRepository interface", () => {
|
||||
const _unused_toDiscourseQueries = (
|
||||
x: ReadRepository
|
||||
): DiscourseQueries => x;
|
||||
});
|
||||
});
|
||||
|
||||
describe("DiscourseInitiativeRepository", () => {
|
||||
it("uses topicsInCategories to find initiative topics", () => {
|
||||
// Given
|
||||
const options = exampleOptions({
|
||||
initiativesCategory: 16,
|
||||
parseCookedHtml: givenParseResponse(examplePartialIniative()),
|
||||
});
|
||||
|
||||
// When
|
||||
const repo = new DiscourseInitiativeRepository(options);
|
||||
repo.initiatives();
|
||||
|
||||
// Then
|
||||
expect(options.queries.topicsInCategories).toBeCalledTimes(1);
|
||||
expect(options.queries.topicsInCategories).toBeCalledWith([16]);
|
||||
});
|
||||
|
||||
it("attempts to get Topic and opening Post for each TopicId found", () => {
|
||||
// Given
|
||||
const options = exampleOptions({
|
||||
topics: [{id: 40}, {id: 41}, {id: 42}],
|
||||
parseCookedHtml: givenParseResponse(examplePartialIniative()),
|
||||
});
|
||||
|
||||
// When
|
||||
const repo = new DiscourseInitiativeRepository(options);
|
||||
repo.initiatives();
|
||||
|
||||
// Then
|
||||
expect(options.queries.topicById).toBeCalledTimes(3);
|
||||
expect(options.queries.topicById).toBeCalledWith(40);
|
||||
expect(options.queries.topicById).toBeCalledWith(41);
|
||||
expect(options.queries.topicById).toBeCalledWith(42);
|
||||
expect(options.queries.postsInTopic).toBeCalledTimes(3);
|
||||
expect(options.queries.postsInTopic).toBeCalledWith(40, 1);
|
||||
expect(options.queries.postsInTopic).toBeCalledWith(41, 1);
|
||||
expect(options.queries.postsInTopic).toBeCalledWith(42, 1);
|
||||
});
|
||||
|
||||
it("filters blacklisted Topics from TopicId found", () => {
|
||||
// Given
|
||||
const options = exampleOptions({
|
||||
topicBlacklist: [41, 50],
|
||||
topics: [{id: 40}, {id: 41}, {id: 42}],
|
||||
parseCookedHtml: givenParseResponse(examplePartialIniative()),
|
||||
});
|
||||
|
||||
// When
|
||||
const repo = new DiscourseInitiativeRepository(options);
|
||||
repo.initiatives();
|
||||
|
||||
// Then
|
||||
expect(options.queries.topicById).toBeCalledTimes(2);
|
||||
expect(options.queries.topicById).toBeCalledWith(40);
|
||||
expect(options.queries.topicById).toBeCalledWith(42);
|
||||
expect(options.queries.postsInTopic).toBeCalledTimes(2);
|
||||
expect(options.queries.postsInTopic).toBeCalledWith(40, 1);
|
||||
expect(options.queries.postsInTopic).toBeCalledWith(42, 1);
|
||||
});
|
||||
|
||||
it("creates Initiatives for matched Topics", () => {
|
||||
// Given
|
||||
const options = exampleOptions({
|
||||
topics: [{id: 40}, {id: 42}],
|
||||
parseCookedHtml: givenParseResponse(
|
||||
examplePartialIniative({
|
||||
references: ["https://example.org/references/included"],
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
// When
|
||||
const repo = new DiscourseInitiativeRepository(options);
|
||||
const initiatives = repo.initiatives();
|
||||
|
||||
// Then
|
||||
expect(initiatives).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"champions": Array [],
|
||||
"completed": false,
|
||||
"contributions": Array [],
|
||||
"dependencies": Array [],
|
||||
"id": Array [
|
||||
"DISCOURSE_TOPIC",
|
||||
"https://foo.bar",
|
||||
"40",
|
||||
],
|
||||
"references": Array [
|
||||
"https://example.org/references/included",
|
||||
],
|
||||
"timestampMs": 1571498171951,
|
||||
"title": "Example initiative",
|
||||
},
|
||||
Object {
|
||||
"champions": Array [],
|
||||
"completed": false,
|
||||
"contributions": Array [],
|
||||
"dependencies": Array [],
|
||||
"id": Array [
|
||||
"DISCOURSE_TOPIC",
|
||||
"https://foo.bar",
|
||||
"42",
|
||||
],
|
||||
"references": Array [
|
||||
"https://example.org/references/included",
|
||||
],
|
||||
"timestampMs": 1571498171951,
|
||||
"title": "Example initiative",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
it("warns when Initiatives fail to parse", () => {
|
||||
// Given
|
||||
const options = exampleOptions({
|
||||
topics: [{id: 40}, {id: 42}],
|
||||
parseCookedHtml: givenParseError("Testing parse error"),
|
||||
});
|
||||
|
||||
// When
|
||||
const repo = new DiscourseInitiativeRepository(options);
|
||||
const initiatives = repo.initiatives();
|
||||
|
||||
// Then
|
||||
expect(initiatives).toEqual([]);
|
||||
expect(console.warn).toHaveBeenCalledWith(
|
||||
dedent`
|
||||
Failed loading [2/2] initiatives:
|
||||
Testing parse error for initiative topic "Example initiative" https://foo.bar/t/40
|
||||
Testing parse error for initiative topic "Example initiative" https://foo.bar/t/42
|
||||
`.trim()
|
||||
);
|
||||
expect(console.warn).toHaveBeenCalledTimes(1);
|
||||
spyWarn().mockReset();
|
||||
});
|
||||
});
|
||||
|
||||
describe("initiativeFromDiscourseTracker", () => {
|
||||
it("assumes values given by the parser", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost();
|
||||
const partial = examplePartialIniative({
|
||||
completed: true,
|
||||
champions: ["https://foo.bar/u/ChampUser"],
|
||||
dependencies: [
|
||||
"https://foo.bar/t/dependency/1",
|
||||
"https://foo.bar/t/dependency/2",
|
||||
"https://foo.bar/t/dependency/3",
|
||||
],
|
||||
references: [
|
||||
"https://foo.bar/t/reference/4",
|
||||
"https://foo.bar/t/reference/5/2",
|
||||
"https://foo.bar/t/reference/6/4",
|
||||
],
|
||||
contributions: [
|
||||
"https://foo.bar/t/contribution/7",
|
||||
"https://foo.bar/t/contribution/8/2",
|
||||
"https://github.com/sourcecred/sourcecred/pull/1416",
|
||||
],
|
||||
});
|
||||
const parser = givenParseResponse(partial);
|
||||
|
||||
// When
|
||||
const initiative = initiativeFromDiscourseTracker(
|
||||
serverUrl,
|
||||
topic,
|
||||
firstPost,
|
||||
parser
|
||||
);
|
||||
|
||||
// Then
|
||||
const actualPartial = {
|
||||
completed: initiative.completed,
|
||||
champions: initiative.champions,
|
||||
dependencies: initiative.dependencies,
|
||||
references: initiative.references,
|
||||
contributions: initiative.contributions,
|
||||
};
|
||||
expect(actualPartial).toEqual(partial);
|
||||
});
|
||||
|
||||
it("assumes title from the topic", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
topic.title = "Different title for test";
|
||||
const firstPost = examplePost();
|
||||
const partial = examplePartialIniative();
|
||||
const parser = givenParseResponse(partial);
|
||||
|
||||
// When
|
||||
const initiative = initiativeFromDiscourseTracker(
|
||||
serverUrl,
|
||||
topic,
|
||||
firstPost,
|
||||
parser
|
||||
);
|
||||
|
||||
// Then
|
||||
expect(initiative.title).toEqual(topic.title);
|
||||
});
|
||||
|
||||
it("assumes timestamp from the post", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost({
|
||||
timestampMs: 901236,
|
||||
});
|
||||
const partial = examplePartialIniative();
|
||||
const parser = givenParseResponse(partial);
|
||||
|
||||
// When
|
||||
const initiative = initiativeFromDiscourseTracker(
|
||||
serverUrl,
|
||||
topic,
|
||||
firstPost,
|
||||
parser
|
||||
);
|
||||
|
||||
// Then
|
||||
expect(initiative.timestampMs).toEqual(firstPost.timestampMs);
|
||||
});
|
||||
|
||||
it("derives the id from topic ID", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic({
|
||||
id: 683,
|
||||
});
|
||||
const firstPost = examplePost({
|
||||
topicId: topic.id,
|
||||
});
|
||||
const partial = examplePartialIniative();
|
||||
const parser = givenParseResponse(partial);
|
||||
|
||||
// When
|
||||
const initiative = initiativeFromDiscourseTracker(
|
||||
serverUrl,
|
||||
topic,
|
||||
firstPost,
|
||||
parser
|
||||
);
|
||||
|
||||
// Then
|
||||
expect(initiative.id).toEqual([
|
||||
"DISCOURSE_TOPIC",
|
||||
serverUrl,
|
||||
String(topic.id),
|
||||
]);
|
||||
});
|
||||
|
||||
it("adds the serverUrl to relative URLs starting with a /", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost();
|
||||
const parser = givenParseResponse(
|
||||
examplePartialIniative({
|
||||
champions: ["/u/ChampUser"],
|
||||
dependencies: ["/t/dependency/1"],
|
||||
references: ["/t/reference/4"],
|
||||
contributions: ["/t/contribution/7"],
|
||||
})
|
||||
);
|
||||
|
||||
// When
|
||||
const initiative = initiativeFromDiscourseTracker(
|
||||
serverUrl,
|
||||
topic,
|
||||
firstPost,
|
||||
parser
|
||||
);
|
||||
|
||||
// Then
|
||||
expect(initiative.champions).toEqual(["https://foo.bar/u/ChampUser"]);
|
||||
expect(initiative.dependencies).toEqual([
|
||||
"https://foo.bar/t/dependency/1",
|
||||
]);
|
||||
expect(initiative.references).toEqual(["https://foo.bar/t/reference/4"]);
|
||||
expect(initiative.contributions).toEqual([
|
||||
"https://foo.bar/t/contribution/7",
|
||||
]);
|
||||
});
|
||||
|
||||
it("throws when post is not associated with this topic", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost({topicId: 15});
|
||||
const parser = givenParseError("SHOULD_NOT_BE_CALLED");
|
||||
|
||||
// When
|
||||
const fn = () =>
|
||||
initiativeFromDiscourseTracker(serverUrl, topic, firstPost, parser);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(
|
||||
'Post 432 is from a different topic for initiative topic "Example initiative" https://foo.bar/t/123'
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when post is not the first in topic", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost({indexWithinTopic: 5});
|
||||
const parser = givenParseError("SHOULD_NOT_BE_CALLED");
|
||||
|
||||
// When
|
||||
const fn = () =>
|
||||
initiativeFromDiscourseTracker(serverUrl, topic, firstPost, parser);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(
|
||||
'Post 432 is not the first post in the topic for initiative topic "Example initiative" https://foo.bar/t/123'
|
||||
);
|
||||
});
|
||||
|
||||
it("extends parse error message with the initiative that caused it", () => {
|
||||
// Given
|
||||
const serverUrl = "https://foo.bar";
|
||||
const topic = exampleTopic();
|
||||
const firstPost = examplePost();
|
||||
const parser = givenParseError("BASE_ERROR_MESSAGE");
|
||||
|
||||
// When
|
||||
const fn = () =>
|
||||
initiativeFromDiscourseTracker(serverUrl, topic, firstPost, parser);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(
|
||||
'BASE_ERROR_MESSAGE for initiative topic "Example initiative" https://foo.bar/t/123'
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,215 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import {DomHandler, DomUtils, Parser} from "htmlparser2";
|
||||
import {type URL} from "./initiative";
|
||||
|
||||
/*
|
||||
All headers are case-insensitive and can be h1-h6.
|
||||
Headers can appear in any order.
|
||||
A matching header for each field must appear exactly once.
|
||||
The expected pattern for a cooked HTML template:
|
||||
|
||||
## Status: complete
|
||||
|
||||
Status value must be in the header, prefixed by "Status:".
|
||||
Either "complete" or "completed". A missing status value,
|
||||
or any other value is considered incomplete.
|
||||
|
||||
## Champions:
|
||||
|
||||
- [@Beanow](/u/beanow)
|
||||
|
||||
Any URLs that appear in the content below the "Champion" or "Champions" header.
|
||||
No filters on user-like types applied here, that's left for after reference detection.
|
||||
|
||||
## Dependencies:
|
||||
|
||||
- [Dependency](/t/topic/123)
|
||||
|
||||
Any URLs that appear in the content below the "Dependency" or "Dependencies" header.
|
||||
|
||||
## References:
|
||||
|
||||
- [Reference](/t/topic/123)
|
||||
|
||||
Any URLs that appear in the content below the "Reference" or "References" header.
|
||||
|
||||
## Contributions:
|
||||
|
||||
- [Contribution](/t/topic/123)
|
||||
|
||||
Any URLs that appear in the content below the "Contribution" or "Contributions" header.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A mapping from an HTML header, to any URLs in the body that follows it.
|
||||
*/
|
||||
type HeaderToURLsMap = Map<string, $ReadOnlyArray<URL>>;
|
||||
|
||||
/**
|
||||
* A partial Iniatiative object, parsed from the Cooked HTML template.
|
||||
*/
|
||||
export type HtmlTemplateInitiativePartial = {|
|
||||
+completed: boolean,
|
||||
+dependencies: $ReadOnlyArray<URL>,
|
||||
+references: $ReadOnlyArray<URL>,
|
||||
+contributions: $ReadOnlyArray<URL>,
|
||||
+champions: $ReadOnlyArray<URL>,
|
||||
|};
|
||||
|
||||
/**
|
||||
* Attempts to parse a cooked HTML body for Initiative data.
|
||||
*
|
||||
* Throws when it doesn't match the template.
|
||||
*/
|
||||
export function parseCookedHtml(
|
||||
cookedHTML: string
|
||||
): HtmlTemplateInitiativePartial {
|
||||
const htu: HeaderToURLsMap = groupURLsByHeader(cookedHTML);
|
||||
const completed = findCompletionStatus(htu);
|
||||
const champions = singleMatch(htu, new RegExp(/^Champions?/i));
|
||||
const contributions = singleMatch(htu, new RegExp(/^Contributions?/i));
|
||||
const dependencies = singleMatch(htu, new RegExp(/^Dependenc(y|ies)/i));
|
||||
const references = singleMatch(htu, new RegExp(/^References?/i));
|
||||
|
||||
const missing = [];
|
||||
if (completed === null) missing.push("status");
|
||||
if (!champions) missing.push("champions");
|
||||
if (!contributions) missing.push("contributions");
|
||||
if (!dependencies) missing.push("dependencies");
|
||||
if (!references) missing.push("references");
|
||||
|
||||
if (
|
||||
completed == null ||
|
||||
champions == null ||
|
||||
contributions == null ||
|
||||
dependencies == null ||
|
||||
references == null
|
||||
) {
|
||||
missing.sort();
|
||||
throw new Error(`Missing or malformed headers ${JSON.stringify(missing)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
completed,
|
||||
dependencies,
|
||||
references,
|
||||
contributions,
|
||||
champions,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes cooked HTML and creates a HeaderToURLsMap.
|
||||
*
|
||||
* Cooked HTML being HTML rendered from Markdown. We're assuming this behaves
|
||||
* a lot like a subset of HTML, even though the option to write HTML manually
|
||||
* exists. For the purpose of parsing Initiative data, we can require just
|
||||
* using Markdown.
|
||||
*
|
||||
* Will throw when there are exact duplicate headers, as this would otherwise
|
||||
* silently merge by header in unexpected ways.
|
||||
*/
|
||||
export function groupURLsByHeader(cookedHTML: string): HeaderToURLsMap {
|
||||
const map: HeaderToURLsMap = new Map();
|
||||
const dom = toDOM(cookedHTML);
|
||||
|
||||
let currentHeader: ?string;
|
||||
for (const rootEl of dom) {
|
||||
switch (rootEl.name) {
|
||||
case "h1":
|
||||
case "h2":
|
||||
case "h3":
|
||||
case "h4":
|
||||
case "h5":
|
||||
case "h6":
|
||||
currentHeader = DomUtils.getText(rootEl);
|
||||
if (map.has(currentHeader)) {
|
||||
throw new Error(
|
||||
`Unsupported duplicate header "${currentHeader}" found`
|
||||
);
|
||||
}
|
||||
// We're also interested in just headers, so make sure an entry exists.
|
||||
map.set(currentHeader, []);
|
||||
break;
|
||||
case "p":
|
||||
case "ul":
|
||||
case "ol":
|
||||
if (currentHeader === undefined) break;
|
||||
const existing = map.get(currentHeader) || [];
|
||||
const anchors = DomUtils.findAll((el) => el.name === "a", [rootEl]).map(
|
||||
(a) => a.attribs.href
|
||||
);
|
||||
map.set(currentHeader, [...existing, ...anchors]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds one "Status:" header, where the value is included in the header itself.
|
||||
*
|
||||
* Returns true when "Status:" is followed by "completed" in the header.
|
||||
* Returns false when "Status:" is followed by any other value.
|
||||
* Returns null when 0 or >1 headers start with "Status:".
|
||||
*/
|
||||
function findCompletionStatus(map: HeaderToURLsMap): boolean | null {
|
||||
const pattern = new RegExp(/^Status:(.*)/i);
|
||||
const headers = Array.from(map.keys())
|
||||
.map((k) => k.trim())
|
||||
.filter((k) => pattern.test(k));
|
||||
|
||||
if (headers.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const matches = headers[0].match(pattern);
|
||||
if (matches == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const completedRE = new RegExp(/^completed?$/i);
|
||||
return completedRE.test(matches[1].trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds one header to match the given RegExp.
|
||||
*
|
||||
* Returns the associated URL[] when exactly 1 header matches.
|
||||
* Returns null when it matches 0 or >1 headers.
|
||||
*/
|
||||
function singleMatch(
|
||||
map: HeaderToURLsMap,
|
||||
pattern: RegExp
|
||||
): $ReadOnlyArray<URL> | null {
|
||||
const headers = Array.from(map.keys()).filter((k) => pattern.test(k.trim()));
|
||||
|
||||
if (headers.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return map.get(headers[0]) || null;
|
||||
}
|
||||
|
||||
function toDOM(cookedHTML: string): Object {
|
||||
// Note: DomHandler is actually synchronous, in spite of the nodeback signature.
|
||||
let dom;
|
||||
const domHandler = new DomHandler((err, result) => {
|
||||
if (err) throw err;
|
||||
dom = result;
|
||||
});
|
||||
|
||||
const htmlParser = new Parser(domHandler);
|
||||
htmlParser.write(cookedHTML);
|
||||
htmlParser.end();
|
||||
|
||||
// The .end() forces data to be flushed, so we know DomHandler calls the callback.
|
||||
// But in case some implementation detail changes, add this error.
|
||||
if (dom === undefined) {
|
||||
throw new Error("DomHandler callback wasn't called after htmlParser.end()");
|
||||
}
|
||||
|
||||
return dom;
|
||||
}
|
||||
|
|
@ -1,310 +0,0 @@
|
|||
// @flow
|
||||
|
||||
import {groupURLsByHeader, parseCookedHtml} from "./htmlTemplate";
|
||||
|
||||
describe("plugins/initiatives/htmlTemplate", () => {
|
||||
describe("parseCookedHtml", () => {
|
||||
const sampleStatusIncomplete = `<h2>Status: Testing</h2>`;
|
||||
const sampleStatusComplete = `<h2>Status: Completed</h2>`;
|
||||
const sampleChampion = `
|
||||
<h2>Champion<a href="https://foo.bar/t/dont-include/10"><sup>?</sup></a>:</h2>
|
||||
<p>
|
||||
<a class="mention" href="/u/ChampUser">@ChampUser</a>
|
||||
</p>
|
||||
`;
|
||||
const sampleDependencies = `
|
||||
<h2>Dependencies:</h2>
|
||||
<ul>
|
||||
<li><a href="https://foo.bar/t/dependency/1">Thing we need</a></li>
|
||||
<li><a href="https://foo.bar/t/dependency/2">Thing we need</a></li>
|
||||
<li><a href="https://foo.bar/t/dependency/3">Thing we need</a></li>
|
||||
</ul>
|
||||
`;
|
||||
const sampleReferences = `
|
||||
<h2>References:</h2>
|
||||
<ul>
|
||||
<li><a href="https://foo.bar/t/reference/4">Some reference</a></li>
|
||||
<li><a href="https://foo.bar/t/reference/5/2">Some reference</a></li>
|
||||
<li><a href="https://foo.bar/t/reference/6/4">Some reference</a></li>
|
||||
</ul>
|
||||
`;
|
||||
const sampleContributions = `
|
||||
<h2>Contributions:</h2>
|
||||
<ul>
|
||||
<li><a href="https://foo.bar/t/contribution/7">Some contribution</a></li>
|
||||
<li><a href="https://foo.bar/t/contribution/8/2">Some contribution</a></li>
|
||||
<li><a href="https://github.com/sourcecred/sourcecred/pull/1416">Some contribution</a></li>
|
||||
</ul>
|
||||
`;
|
||||
|
||||
it("handles an example text", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const partial = parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(partial).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"champions": Array [
|
||||
"/u/ChampUser",
|
||||
],
|
||||
"completed": false,
|
||||
"contributions": Array [
|
||||
"https://foo.bar/t/contribution/7",
|
||||
"https://foo.bar/t/contribution/8/2",
|
||||
"https://github.com/sourcecred/sourcecred/pull/1416",
|
||||
],
|
||||
"dependencies": Array [
|
||||
"https://foo.bar/t/dependency/1",
|
||||
"https://foo.bar/t/dependency/2",
|
||||
"https://foo.bar/t/dependency/3",
|
||||
],
|
||||
"references": Array [
|
||||
"https://foo.bar/t/reference/4",
|
||||
"https://foo.bar/t/reference/5/2",
|
||||
"https://foo.bar/t/reference/6/4",
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it("considers blank status incomplete", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
<h1>Example initiative</h1>
|
||||
<h2>Status:</h2>
|
||||
<h2>Champion:</h2>
|
||||
<h2>Dependencies:</h2>
|
||||
<h2>References:</h2>
|
||||
<h2>Contributions:</h2>
|
||||
`;
|
||||
|
||||
// When
|
||||
const partial = parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(partial.completed).toEqual(false);
|
||||
});
|
||||
|
||||
it("throws for missing all headers", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
<h1>Example initiative</h1>
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(
|
||||
`Missing or malformed headers ["champions","contributions","dependencies","references","status"]`
|
||||
);
|
||||
});
|
||||
|
||||
it("throws for missing status header", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["status"]`);
|
||||
});
|
||||
|
||||
it("throws for missing champions header", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["champions"]`);
|
||||
});
|
||||
|
||||
it("throws for missing dependencies header", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleChampion}
|
||||
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["dependencies"]`);
|
||||
});
|
||||
|
||||
it("throws for missing references header", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["references"]`);
|
||||
});
|
||||
|
||||
it("throws for missing contributions header", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["contributions"]`);
|
||||
});
|
||||
|
||||
it("throws for conflicting status headers", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleStatusComplete}
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Missing or malformed headers ["status"]`);
|
||||
});
|
||||
|
||||
it("throws for duplicate headers", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
${sampleStatusIncomplete}
|
||||
${sampleChampion}
|
||||
${sampleDependencies}
|
||||
${sampleDependencies}
|
||||
${sampleReferences}
|
||||
${sampleContributions}
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => parseCookedHtml(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(`Unsupported duplicate header "Dependencies:" found`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupURLsByHeader", () => {
|
||||
it("handles an example text", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
<h1>This is a title</h1>
|
||||
<p>
|
||||
Things to talk about.
|
||||
<a href="https://foo.bar/1">With links</a>
|
||||
</p>
|
||||
<a href="https://foo.bar/baz">Seems unmarkdownly formatted</a>
|
||||
<h2>Some <i>funky</i> section:</h2>
|
||||
<p>
|
||||
<a href="https://foo.bar/2">With</a>
|
||||
<strong><a href="https://foo.bar/3">More</a></strong>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://foo.bar/4">Links</a>
|
||||
</p>
|
||||
<h2>Listed things<a href="https://foo.bar/t/dont-include/10"><sup>?</sup></a>:</h2>
|
||||
<ul>
|
||||
<li><a href="https://foo.bar/5">Yet</a></li>
|
||||
<li><a href="https://foo.bar/6">More</a></li>
|
||||
<li><a href="https://foo.bar/7">Links</a></li>
|
||||
</ul>
|
||||
<h2>Ordered things:</h2>
|
||||
<ol>
|
||||
<li><a href="https://foo.bar/8">Yet</a></li>
|
||||
<li><a href="https://foo.bar/9">More</a></li>
|
||||
<li><a href="https://foo.bar/10">Links</a></li>
|
||||
</ol>
|
||||
`;
|
||||
|
||||
// When
|
||||
const map = groupURLsByHeader(sample);
|
||||
|
||||
// Then
|
||||
expect(map).toMatchInlineSnapshot(`
|
||||
Map {
|
||||
"This is a title" => Array [
|
||||
"https://foo.bar/1",
|
||||
],
|
||||
"Some funky section:" => Array [
|
||||
"https://foo.bar/2",
|
||||
"https://foo.bar/3",
|
||||
"https://foo.bar/4",
|
||||
],
|
||||
"Listed things?:" => Array [
|
||||
"https://foo.bar/5",
|
||||
"https://foo.bar/6",
|
||||
"https://foo.bar/7",
|
||||
],
|
||||
"Ordered things:" => Array [
|
||||
"https://foo.bar/8",
|
||||
"https://foo.bar/9",
|
||||
"https://foo.bar/10",
|
||||
],
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it("throws for duplicate headers", () => {
|
||||
// Given
|
||||
const sample = `
|
||||
<h1>This is a title</h1>
|
||||
<h1>This is a title</h1>
|
||||
`;
|
||||
|
||||
// When
|
||||
const fn = () => groupURLsByHeader(sample);
|
||||
|
||||
// Then
|
||||
expect(fn).toThrow(
|
||||
`Unsupported duplicate header "This is a title" found`
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue