From 72f2d0254505f6c41cb8c0adba2c3c8c7050bff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Sun, 6 Oct 2019 15:25:48 -0600 Subject: [PATCH] Add post contents to the Discourse mirror This modifies the Discourse fetcher and mirror so that we now keep post contents around, thus enabling future reference detection (and other things). The post contents are stored and provided as retrieved from the API, which is in "cooked" HTML form. Test plan: Unit tests and snapshots updated. Observe that the snapshots now include Discourse post contents. This is progress towards [Discourse reference and mention detection][1]. [1]: https://discourse.sourcecred.io/t/discourse-reference-mention-detection/270 --- .../__snapshots__/fetch.test.js.snap | 30 +++++++++++++++++++ src/plugins/discourse/createGraph.test.js | 4 +++ src/plugins/discourse/fetch.js | 3 ++ src/plugins/discourse/mirror.js | 14 ++++++--- src/plugins/discourse/mirror.test.js | 7 ++++- 5 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/plugins/discourse/__snapshots__/fetch.test.js.snap b/src/plugins/discourse/__snapshots__/fetch.test.js.snap index 96f9076..ce458ff 100644 --- a/src/plugins/discourse/__snapshots__/fetch.test.js.snap +++ b/src/plugins/discourse/__snapshots__/fetch.test.js.snap @@ -3,6 +3,7 @@ exports[`plugins/discourse/fetch snapshot testing loads a particular post from snapshot 1`] = ` Object { "authorUsername": "d11", + "cooked": "

This is a test post.

", "id": 14, "indexWithinTopic": 1, "replyToPostIndex": null, @@ -16,6 +17,7 @@ Object { "posts": Array [ Object { "authorUsername": "d11", + "cooked": "

This is a test post.

", "id": 14, "indexWithinTopic": 1, "replyToPostIndex": null, @@ -24,6 +26,7 @@ Object { }, Object { "authorUsername": "dl-proto", + "cooked": "

Ah, an excellent post. I will like it and reply.

", "id": 16, "indexWithinTopic": 2, "replyToPostIndex": null, @@ -32,6 +35,7 @@ Object { }, Object { "authorUsername": "d11", + "cooked": "

Adding another post, after having deleted a whole thread.

", "id": 22, "indexWithinTopic": 3, "replyToPostIndex": null, @@ -52,6 +56,7 @@ exports[`plugins/discourse/fetch snapshot testing loads latest posts from snapsh Array [ Object { "authorUsername": "d11", + "cooked": "

Adding another post, after having deleted a whole thread.

", "id": 22, "indexWithinTopic": 3, "replyToPostIndex": null, @@ -60,6 +65,15 @@ Array [ }, Object { "authorUsername": "d11", + "cooked": " +

Excellent link. I’ve quoted you.

", "id": 18, "indexWithinTopic": 2, "replyToPostIndex": null, @@ -68,6 +82,8 @@ Array [ }, Object { "authorUsername": "dl-proto", + "cooked": "

Here is a link to another discourse thread: My First Test Post

+

Here is a link to a GitHub issue: https://github.com/sourcecred-test/example-github/issues/1

", "id": 17, "indexWithinTopic": 1, "replyToPostIndex": null, @@ -76,6 +92,7 @@ Array [ }, Object { "authorUsername": "dl-proto", + "cooked": "

Ah, an excellent post. I will like it and reply.

", "id": 16, "indexWithinTopic": 2, "replyToPostIndex": null, @@ -84,6 +101,7 @@ Array [ }, Object { "authorUsername": "d11", + "cooked": "

This is a test post.

", "id": 14, "indexWithinTopic": 1, "replyToPostIndex": null, @@ -92,6 +110,17 @@ Array [ }, Object { "authorUsername": "system", + "cooked": "

This is a test instance.
+The first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It’s important!

+

Edit this into a brief description of your community:

+ +

+

You may want to close this topic via the admin \\":wrench:\\" (at the upper right and bottom), so that replies don’t pile up on an announcement.

", "id": 10, "indexWithinTopic": 1, "replyToPostIndex": null, @@ -100,6 +129,7 @@ Array [ }, Object { "authorUsername": "system", + "cooked": "

Discussion about this site, its organization, how it works, and how we can improve it.

", "id": 1, "indexWithinTopic": 1, "replyToPostIndex": null, diff --git a/src/plugins/discourse/createGraph.test.js b/src/plugins/discourse/createGraph.test.js index d35e259..4be882b 100644 --- a/src/plugins/discourse/createGraph.test.js +++ b/src/plugins/discourse/createGraph.test.js @@ -82,6 +82,7 @@ describe("plugins/discourse/createGraph", () => { replyToPostIndex: null, timestampMs: 0, authorUsername: "decentralion", + cooked: "

Hello

", }; const post2 = { id: 2, @@ -92,6 +93,7 @@ describe("plugins/discourse/createGraph", () => { replyToPostIndex: null, timestampMs: 1, authorUsername: "wchargin", + cooked: "

Hello

", }; const post3 = { id: 3, @@ -100,6 +102,7 @@ describe("plugins/discourse/createGraph", () => { replyToPostIndex: 2, timestampMs: 1, authorUsername: "mzargham", + cooked: "

Hello

", }; const likes: $ReadOnlyArray = [ {timestampMs: 3, username: "mzargham", postId: 2}, @@ -171,6 +174,7 @@ describe("plugins/discourse/createGraph", () => { replyToPostIndex: null, timestampMs: 0, authorUsername: "decentralion", + cooked: "

Hello

", }; const data = new MockData([], [post], []); const url = "https://foo"; diff --git a/src/plugins/discourse/fetch.js b/src/plugins/discourse/fetch.js index 8aee2e4..5660de4 100644 --- a/src/plugins/discourse/fetch.js +++ b/src/plugins/discourse/fetch.js @@ -37,6 +37,8 @@ export type Post = {| +replyToPostIndex: number | null, +timestampMs: number, +authorUsername: string, + // The post HTML for rendering. + +cooked: string, |}; export type TopicWithPosts = {| @@ -236,6 +238,7 @@ function parsePost(json: any): Post { replyToPostIndex: json.reply_to_post_number, topicId: json.topic_id, authorUsername: json.username, + cooked: json.cooked, }; } diff --git a/src/plugins/discourse/mirror.js b/src/plugins/discourse/mirror.js index 8f32c22..9187f57 100644 --- a/src/plugins/discourse/mirror.js +++ b/src/plugins/discourse/mirror.js @@ -15,7 +15,7 @@ import { // The version should be bumped any time the database schema is changed, // so that the cache will be properly invalidated. -const VERSION = "discourse_mirror_v3"; +const VERSION = "discourse_mirror_v4"; /** * An interface for retrieving all of the Discourse data at once. @@ -165,6 +165,7 @@ export class Mirror implements DiscourseData { topic_id INTEGER NOT NULL, index_within_topic INTEGER NOT NULL, reply_to_post_index INTEGER, + cooked TEXT NOT NULL, FOREIGN KEY(topic_id) REFERENCES topics(id), FOREIGN KEY(author_username) REFERENCES users(username) ) @@ -214,7 +215,8 @@ export class Mirror implements DiscourseData { author_username, topic_id, index_within_topic, - reply_to_post_index + reply_to_post_index, + cooked FROM posts` ) .all() @@ -225,6 +227,7 @@ export class Mirror implements DiscourseData { topicId: x.topic_id, indexWithinTopic: x.index_within_topic, replyToPostIndex: x.reply_to_post_index, + cooked: x.cooked, })); } @@ -283,14 +286,16 @@ export class Mirror implements DiscourseData { author_username, topic_id, index_within_topic, - reply_to_post_index + reply_to_post_index, + cooked ) VALUES ( :id, :timestamp_ms, :author_username, :topic_id, :index_within_topic, - :reply_to_post_index + :reply_to_post_index, + :cooked ) ` ); @@ -306,6 +311,7 @@ export class Mirror implements DiscourseData { index_within_topic: post.indexWithinTopic, topic_id: post.topicId, author_username: post.authorUsername, + cooked: post.cooked, }); } catch (e) { const url = `${serverUrl}/t/${post.topicId}/${post.indexWithinTopic}`; diff --git a/src/plugins/discourse/mirror.test.js b/src/plugins/discourse/mirror.test.js index 20417f7..572f52d 100644 --- a/src/plugins/discourse/mirror.test.js +++ b/src/plugins/discourse/mirror.test.js @@ -23,6 +23,7 @@ type PostInfo = {| +replyToPostIndex: number | null, +topicId: number, +authorUsername: string, + +cooked: string, |}; class MockFetcher implements Discourse { @@ -101,6 +102,7 @@ class MockFetcher implements Discourse { topicId, indexWithinTopic, authorUsername, + cooked, } = postInfo; return { id, @@ -109,13 +111,15 @@ class MockFetcher implements Discourse { topicId, indexWithinTopic, authorUsername, + cooked, }; } addPost( topicId: TopicId, replyToNumber: number | null, - username?: string + username?: string, + cooked?: string ): PostId { const postId = this._latestPostId++; this._latestTopicId = Math.max(topicId, this._latestTopicId); @@ -132,6 +136,7 @@ class MockFetcher implements Discourse { replyToPostIndex: replyToNumber, topicId: topicId, authorUsername: NullUtil.orElse(username, "credbot"), + cooked: NullUtil.orElse(cooked, "

Hello World

"), }; this._posts.set(postId, postInfo); return postId;