mirror of
https://github.com/status-im/sourcecred.git
synced 2025-01-11 13:14:28 +00:00
Add post contents to the Discourse mirror
This modifies the Discourse fetcher and mirror so that we now keep post contents around, thus enabling future reference detection (and other things). The post contents are stored and provided as retrieved from the API, which is in "cooked" HTML form. Test plan: Unit tests and snapshots updated. Observe that the snapshots now include Discourse post contents. This is progress towards [Discourse reference and mention detection][1]. [1]: https://discourse.sourcecred.io/t/discourse-reference-mention-detection/270
This commit is contained in:
Dandelion Mané
parent
65edd01dcf
commit
72f2d02545
@ -3,6 +3,7 @@
|
||||
exports[`plugins/discourse/fetch snapshot testing loads a particular post from snapshot 1`] = `
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<p>This is a test post.</p>",
|
||||
"id": 14,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
@ -16,6 +17,7 @@ Object {
|
||||
"posts": Array [
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<p>This is a test post.</p>",
|
||||
"id": 14,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
@ -24,6 +26,7 @@ Object {
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "dl-proto",
|
||||
"cooked": "<p>Ah, an excellent post. I will like it and reply.</p>",
|
||||
"id": 16,
|
||||
"indexWithinTopic": 2,
|
||||
"replyToPostIndex": null,
|
||||
@ -32,6 +35,7 @@ Object {
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<p>Adding another post, after having deleted a whole thread.</p>",
|
||||
"id": 22,
|
||||
"indexWithinTopic": 3,
|
||||
"replyToPostIndex": null,
|
||||
@ -52,6 +56,7 @@ exports[`plugins/discourse/fetch snapshot testing loads latest posts from snapsh
|
||||
Array [
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<p>Adding another post, after having deleted a whole thread.</p>",
|
||||
"id": 22,
|
||||
"indexWithinTopic": 3,
|
||||
"replyToPostIndex": null,
|
||||
@ -60,6 +65,15 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<aside class=\\"quote no-group\\" data-post=\\"1\\" data-topic=\\"13\\">
|
||||
<div class=\\"title\\">
|
||||
<div class=\\"quote-controls\\"></div>
|
||||
<img alt width=\\"20\\" height=\\"20\\" src=\\"https://avatars.discourse.org/v4/letter/d/8797f3/40.png\\" class=\\"avatar\\"> dl-proto:</div>
|
||||
<blockquote>
|
||||
<p>Here is a link to another discourse thread:</p>
|
||||
</blockquote>
|
||||
</aside>
|
||||
<p>Excellent link. I’ve quoted you.</p>",
|
||||
"id": 18,
|
||||
"indexWithinTopic": 2,
|
||||
"replyToPostIndex": null,
|
||||
@ -68,6 +82,8 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "dl-proto",
|
||||
"cooked": "<p>Here is a link to another discourse thread: <a href=\\"https://sourcecred-test.discourse.group/t/my-first-test-post/11/3\\" class=\\"inline-onebox\\">My First Test Post</a></p>
|
||||
<p>Here is a link to a GitHub issue: <a href=\\"https://github.com/sourcecred-test/example-github/issues/1\\" rel=\\"nofollow noopener\\">https://github.com/sourcecred-test/example-github/issues/1</a></p>",
|
||||
"id": 17,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
@ -76,6 +92,7 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "dl-proto",
|
||||
"cooked": "<p>Ah, an excellent post. I will like it and reply.</p>",
|
||||
"id": 16,
|
||||
"indexWithinTopic": 2,
|
||||
"replyToPostIndex": null,
|
||||
@ -84,6 +101,7 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "d11",
|
||||
"cooked": "<p>This is a test post.</p>",
|
||||
"id": 14,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
@ -92,6 +110,17 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "system",
|
||||
"cooked": "<p>This is a test instance.<br>
|
||||
The first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It’s important!</p>
|
||||
<p><strong>Edit this</strong> into a brief description of your community:</p>
|
||||
<ul>
|
||||
<li>Who is it for?</li>
|
||||
<li>What can they find here?</li>
|
||||
<li>Why should they come here?</li>
|
||||
<li>Where can they read more (links, resources, etc)?</li>
|
||||
</ul>
|
||||
<p><img src=\\"https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif\\" width=\\"508\\" height=\\"106\\"></p>
|
||||
<p>You may want to close this topic via the admin <img src=\\"https://sjc3.discourse-cdn.com/free1/images/emoji/twitter/wrench.png?v=9\\" title=\\":wrench:\\" class=\\"emoji\\" alt=\\":wrench:\\"> (at the upper right and bottom), so that replies don’t pile up on an announcement.</p>",
|
||||
"id": 10,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
@ -100,6 +129,7 @@ Array [
|
||||
},
|
||||
Object {
|
||||
"authorUsername": "system",
|
||||
"cooked": "<p>Discussion about this site, its organization, how it works, and how we can improve it.</p>",
|
||||
"id": 1,
|
||||
"indexWithinTopic": 1,
|
||||
"replyToPostIndex": null,
|
||||
|
||||
@ -82,6 +82,7 @@ describe("plugins/discourse/createGraph", () => {
|
||||
replyToPostIndex: null,
|
||||
timestampMs: 0,
|
||||
authorUsername: "decentralion",
|
||||
cooked: "<h1>Hello</h1>",
|
||||
};
|
||||
const post2 = {
|
||||
id: 2,
|
||||
@ -92,6 +93,7 @@ describe("plugins/discourse/createGraph", () => {
|
||||
replyToPostIndex: null,
|
||||
timestampMs: 1,
|
||||
authorUsername: "wchargin",
|
||||
cooked: "<h1>Hello</h1>",
|
||||
};
|
||||
const post3 = {
|
||||
id: 3,
|
||||
@ -100,6 +102,7 @@ describe("plugins/discourse/createGraph", () => {
|
||||
replyToPostIndex: 2,
|
||||
timestampMs: 1,
|
||||
authorUsername: "mzargham",
|
||||
cooked: "<h1>Hello</h1>",
|
||||
};
|
||||
const likes: $ReadOnlyArray<LikeAction> = [
|
||||
{timestampMs: 3, username: "mzargham", postId: 2},
|
||||
@ -171,6 +174,7 @@ describe("plugins/discourse/createGraph", () => {
|
||||
replyToPostIndex: null,
|
||||
timestampMs: 0,
|
||||
authorUsername: "decentralion",
|
||||
cooked: "<h1>Hello</h1>",
|
||||
};
|
||||
const data = new MockData([], [post], []);
|
||||
const url = "https://foo";
|
||||
|
||||
@ -37,6 +37,8 @@ export type Post = {|
|
||||
+replyToPostIndex: number | null,
|
||||
+timestampMs: number,
|
||||
+authorUsername: string,
|
||||
// The post HTML for rendering.
|
||||
+cooked: string,
|
||||
|};
|
||||
|
||||
export type TopicWithPosts = {|
|
||||
@ -236,6 +238,7 @@ function parsePost(json: any): Post {
|
||||
replyToPostIndex: json.reply_to_post_number,
|
||||
topicId: json.topic_id,
|
||||
authorUsername: json.username,
|
||||
cooked: json.cooked,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ import {
|
||||
|
||||
// The version should be bumped any time the database schema is changed,
|
||||
// so that the cache will be properly invalidated.
|
||||
const VERSION = "discourse_mirror_v3";
|
||||
const VERSION = "discourse_mirror_v4";
|
||||
|
||||
/**
|
||||
* An interface for retrieving all of the Discourse data at once.
|
||||
@ -165,6 +165,7 @@ export class Mirror implements DiscourseData {
|
||||
topic_id INTEGER NOT NULL,
|
||||
index_within_topic INTEGER NOT NULL,
|
||||
reply_to_post_index INTEGER,
|
||||
cooked TEXT NOT NULL,
|
||||
FOREIGN KEY(topic_id) REFERENCES topics(id),
|
||||
FOREIGN KEY(author_username) REFERENCES users(username)
|
||||
)
|
||||
@ -214,7 +215,8 @@ export class Mirror implements DiscourseData {
|
||||
author_username,
|
||||
topic_id,
|
||||
index_within_topic,
|
||||
reply_to_post_index
|
||||
reply_to_post_index,
|
||||
cooked
|
||||
FROM posts`
|
||||
)
|
||||
.all()
|
||||
@ -225,6 +227,7 @@ export class Mirror implements DiscourseData {
|
||||
topicId: x.topic_id,
|
||||
indexWithinTopic: x.index_within_topic,
|
||||
replyToPostIndex: x.reply_to_post_index,
|
||||
cooked: x.cooked,
|
||||
}));
|
||||
}
|
||||
|
||||
@ -283,14 +286,16 @@ export class Mirror implements DiscourseData {
|
||||
author_username,
|
||||
topic_id,
|
||||
index_within_topic,
|
||||
reply_to_post_index
|
||||
reply_to_post_index,
|
||||
cooked
|
||||
) VALUES (
|
||||
:id,
|
||||
:timestamp_ms,
|
||||
:author_username,
|
||||
:topic_id,
|
||||
:index_within_topic,
|
||||
:reply_to_post_index
|
||||
:reply_to_post_index,
|
||||
:cooked
|
||||
)
|
||||
`
|
||||
);
|
||||
@ -306,6 +311,7 @@ export class Mirror implements DiscourseData {
|
||||
index_within_topic: post.indexWithinTopic,
|
||||
topic_id: post.topicId,
|
||||
author_username: post.authorUsername,
|
||||
cooked: post.cooked,
|
||||
});
|
||||
} catch (e) {
|
||||
const url = `${serverUrl}/t/${post.topicId}/${post.indexWithinTopic}`;
|
||||
|
||||
@ -23,6 +23,7 @@ type PostInfo = {|
|
||||
+replyToPostIndex: number | null,
|
||||
+topicId: number,
|
||||
+authorUsername: string,
|
||||
+cooked: string,
|
||||
|};
|
||||
|
||||
class MockFetcher implements Discourse {
|
||||
@ -101,6 +102,7 @@ class MockFetcher implements Discourse {
|
||||
topicId,
|
||||
indexWithinTopic,
|
||||
authorUsername,
|
||||
cooked,
|
||||
} = postInfo;
|
||||
return {
|
||||
id,
|
||||
@ -109,13 +111,15 @@ class MockFetcher implements Discourse {
|
||||
topicId,
|
||||
indexWithinTopic,
|
||||
authorUsername,
|
||||
cooked,
|
||||
};
|
||||
}
|
||||
|
||||
addPost(
|
||||
topicId: TopicId,
|
||||
replyToNumber: number | null,
|
||||
username?: string
|
||||
username?: string,
|
||||
cooked?: string
|
||||
): PostId {
|
||||
const postId = this._latestPostId++;
|
||||
this._latestTopicId = Math.max(topicId, this._latestTopicId);
|
||||
@ -132,6 +136,7 @@ class MockFetcher implements Discourse {
|
||||
replyToPostIndex: replyToNumber,
|
||||
topicId: topicId,
|
||||
authorUsername: NullUtil.orElse(username, "credbot"),
|
||||
cooked: NullUtil.orElse(cooked, "<h1>Hello World</h1>"),
|
||||
};
|
||||
this._posts.set(postId, postInfo);
|
||||
return postId;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user