Discourse: make topicWithPosts fetch all posts (#1455)

Previously it would only consider page 1.
Now we're walking through all pages, as this
is a much more effective way of discovering
all posts.
This commit is contained in:
Robin van Boven 2019-11-16 13:59:18 +01:00 committed by GitHub
parent 23f1db6ce4
commit 623c362246
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1721 additions and 6 deletions

View File

@ -55,6 +55,250 @@ Object {
}
`;
exports[`plugins/discourse/fetch snapshot testing loads a topic with pagination from snapshot 1`] = `
Object {
"posts": Array [
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>This topic will have 2 pages.<br>
Needing at least &gt; 20 posts.</p>",
"id": 34,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1573839733554,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#2</span>
</h2>",
"id": 35,
"indexWithinTopic": 2,
"replyToPostIndex": null,
"timestampMs": 1573839810871,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#3</span>
</h2>",
"id": 36,
"indexWithinTopic": 3,
"replyToPostIndex": null,
"timestampMs": 1573839814701,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#4</span>
</h2>",
"id": 37,
"indexWithinTopic": 4,
"replyToPostIndex": null,
"timestampMs": 1573839818330,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#5</span>
</h2>",
"id": 38,
"indexWithinTopic": 5,
"replyToPostIndex": null,
"timestampMs": 1573839821821,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#6</span>
</h2>",
"id": 39,
"indexWithinTopic": 6,
"replyToPostIndex": null,
"timestampMs": 1573839825601,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#7</span>
</h2>",
"id": 40,
"indexWithinTopic": 7,
"replyToPostIndex": null,
"timestampMs": 1573839828825,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#8</span>
</h2>",
"id": 41,
"indexWithinTopic": 8,
"replyToPostIndex": null,
"timestampMs": 1573839832023,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#9</span>
</h2>",
"id": 42,
"indexWithinTopic": 9,
"replyToPostIndex": null,
"timestampMs": 1573839835298,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#10</span>
</h2>",
"id": 43,
"indexWithinTopic": 10,
"replyToPostIndex": null,
"timestampMs": 1573839840261,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#11</span>
</h2>",
"id": 44,
"indexWithinTopic": 11,
"replyToPostIndex": null,
"timestampMs": 1573839843418,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#12</span>
</h2>",
"id": 45,
"indexWithinTopic": 12,
"replyToPostIndex": null,
"timestampMs": 1573839846537,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#13</span>
</h2>",
"id": 46,
"indexWithinTopic": 13,
"replyToPostIndex": null,
"timestampMs": 1573839849459,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#14</span>
</h2>",
"id": 47,
"indexWithinTopic": 14,
"replyToPostIndex": null,
"timestampMs": 1573839852186,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#15</span>
</h2>",
"id": 48,
"indexWithinTopic": 15,
"replyToPostIndex": null,
"timestampMs": 1573839855068,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#16</span>
</h2>",
"id": 49,
"indexWithinTopic": 16,
"replyToPostIndex": null,
"timestampMs": 1573839857787,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#17</span>
</h2>",
"id": 50,
"indexWithinTopic": 17,
"replyToPostIndex": null,
"timestampMs": 1573839860730,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#18</span>
</h2>",
"id": 51,
"indexWithinTopic": 18,
"replyToPostIndex": null,
"timestampMs": 1573839863576,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#19</span>
</h2>",
"id": 52,
"indexWithinTopic": 19,
"replyToPostIndex": null,
"timestampMs": 1573839866548,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#20</span>
</h2>",
"id": 53,
"indexWithinTopic": 20,
"replyToPostIndex": null,
"timestampMs": 1573839869555,
"topicId": 26,
},
Object {
"authorUsername": "beanow.sc-test",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>
<h2>Post <span class=\\"hashtag\\">#21</span>
</h2>",
"id": 54,
"indexWithinTopic": 21,
"replyToPostIndex": null,
"timestampMs": 1573839872244,
"topicId": 26,
},
],
"topic": Object {
"authorUsername": "beanow.sc-test",
"categoryId": 6,
"id": 26,
"timestampMs": 1573839733506,
"title": "Big Filler Topic 1",
},
}
`;
exports[`plugins/discourse/fetch snapshot testing loads latest posts from snapshot 1`] = `
Array [
Object {

View File

@ -60,12 +60,7 @@ export type Post = {|
export type TopicWithPosts = {|
+topic: TopicView,
// Not guaranteed to contain all the Posts in the topic—clients will need to
// manually request some posts. The raw response actually includes a list of
// all the PostIds in the topic, but for now we don't use them.
//
// We do use these Posts though, as it allows us to save requesting them all
// individually.
// Guaranteed to contain all the Posts in the topic.
+posts: $ReadOnlyArray<Post>,
|};
@ -198,6 +193,7 @@ export class Fetcher implements Discourse {
}
failForNotOk(response);
const json = await response.json();
const {posts_count: postCount} = json;
let posts = json.post_stream.posts.map(parsePost);
const topic: TopicView = {
id: json.id,
@ -206,6 +202,19 @@ export class Fetcher implements Discourse {
timestampMs: Date.parse(json.created_at),
authorUsername: json.details.created_by.username,
};
// This shouldn't could cause infinite loops when the API is weird.
// As requesting pages beyond the last page will produce a 404.
// Pagination here is 1-based, and we already had page 1.
let page = 2;
while (postCount > posts.length) {
const resNext = await this._fetch(`/t/${id}.json?page=${page}`);
failForNotOk(resNext);
const subPosts = (await resNext.json()).post_stream.posts.map(parsePost);
posts = [...posts, ...subPosts];
page++;
}
return {topic, posts};
}

View File

@ -15,6 +15,9 @@ describe("plugins/discourse/fetch", () => {
it("loads a particular topic from snapshot", async () => {
expect(await snapshotFetcher().topicWithPosts(11)).toMatchSnapshot();
});
it("loads a topic with pagination from snapshot", async () => {
expect(await snapshotFetcher().topicWithPosts(26)).toMatchSnapshot();
});
it("loads a particular post from snapshot", async () => {
expect(await snapshotFetcher().post(14)).toMatchSnapshot();
});

View File

@ -0,0 +1,373 @@
{
"post_stream": {
"posts": [
{
"id": 54,
"name": "Beanow",
"username": "beanow.sc-test",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png",
"created_at": "2019-11-15T17:44:32.244Z",
"cooked": "<p>The quick brown fox jumps over the lazy dog.</p>\n<h2>Post <span class=\"hashtag\">#21</span>\n</h2>",
"post_number": 21,
"post_type": 1,
"updated_at": "2019-11-15T17:44:32.244Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 1,
"readers_count": 0,
"score": 0.2,
"yours": false,
"topic_id": 26,
"topic_slug": "big-filler-topic-1",
"display_username": "Beanow",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"read": true,
"user_title": null,
"actions_summary": [],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 5,
"hidden": false,
"trust_level": 1,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
}
],
"stream": [
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54
]
},
"timeline_lookup": [
[
1,
0
]
],
"suggested_topics": [
{
"id": 7,
"title": "Welcome to Discourse",
"fancy_title": "Welcome to Discourse",
"slug": "welcome-to-discourse",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": "https://sjc2.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif",
"created_at": "2019-04-02T20:20:12.879Z",
"last_posted_at": "2019-04-02T20:20:12.933Z",
"bumped": true,
"bumped_at": "2019-04-10T23:16:19.399Z",
"unseen": false,
"pinned": true,
"unpinned": null,
"excerpt": "This is a test instance. \nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. Its important! \nEdit this into a brief description of your community: \n\nWho i&hellip;",
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 10,
"category_id": 1,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": -1,
"username": "system",
"name": "system",
"avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png"
}
}
]
},
{
"id": 27,
"title": "Filler Topic #1",
"fancy_title": "Filler Topic #1",
"slug": "filler-topic-1",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": null,
"created_at": "2019-11-15T17:49:19.140Z",
"last_posted_at": "2019-11-15T17:49:19.198Z",
"bumped": true,
"bumped_at": "2019-11-15T17:49:19.198Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 2,
"category_id": 5,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
}
}
]
},
{
"id": 28,
"title": "Filler Topic #2",
"fancy_title": "Filler Topic #2",
"slug": "filler-topic-2",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": null,
"created_at": "2019-11-15T17:50:33.177Z",
"last_posted_at": "2019-11-15T17:50:33.231Z",
"bumped": true,
"bumped_at": "2019-11-15T17:50:33.231Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 2,
"category_id": 5,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
}
}
]
},
{
"id": 44,
"title": "Filler Topic #18",
"fancy_title": "Filler Topic #18",
"slug": "filler-topic-18",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": null,
"created_at": "2019-11-15T17:55:02.978Z",
"last_posted_at": "2019-11-15T17:55:03.032Z",
"bumped": true,
"bumped_at": "2019-11-15T17:55:03.032Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 1,
"category_id": 5,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
}
}
]
},
{
"id": 45,
"title": "Filler Topic #19",
"fancy_title": "Filler Topic #19",
"slug": "filler-topic-19",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": null,
"created_at": "2019-11-15T17:55:09.297Z",
"last_posted_at": "2019-11-15T17:55:09.355Z",
"bumped": true,
"bumped_at": "2019-11-15T17:55:09.355Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 1,
"category_id": 5,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
}
}
]
}
],
"id": 26,
"title": "Big Filler Topic 1",
"fancy_title": "Big Filler Topic 1",
"posts_count": 21,
"created_at": "2019-11-15T17:42:13.506Z",
"views": 2,
"reply_count": 0,
"like_count": 0,
"last_posted_at": "2019-11-15T17:44:32.244Z",
"visible": true,
"closed": false,
"archived": false,
"has_summary": false,
"archetype": "regular",
"slug": "big-filler-topic-1",
"category_id": 6,
"word_count": 231,
"deleted_at": null,
"user_id": 5,
"featured_link": null,
"pinned_globally": false,
"pinned_at": null,
"pinned_until": null,
"image_url": null,
"draft": null,
"draft_key": "topic_26",
"draft_sequence": null,
"unpinned": null,
"pinned": false,
"current_post_number": 1,
"highest_post_number": 21,
"deleted_by": null,
"actions_summary": [
{
"id": 4,
"count": 0,
"hidden": false,
"can_act": false
},
{
"id": 8,
"count": 0,
"hidden": false,
"can_act": false
},
{
"id": 7,
"count": 0,
"hidden": false,
"can_act": false
}
],
"chunk_size": 20,
"bookmarked": null,
"topic_timer": null,
"message_bus_last_id": 21,
"participant_count": 1,
"show_read_indicator": false,
"tags_disable_ads": false,
"details": {
"notification_level": 1,
"participants": [
{
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png",
"post_count": 21,
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_color": null,
"primary_group_flair_bg_color": null
}
],
"created_by": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
},
"last_poster": {
"id": 5,
"username": "beanow.sc-test",
"name": "Beanow",
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
}
}
}

View File

@ -31,3 +31,7 @@ fetch "/t/11.json"
fetch "/t/21.json"
fetch "/posts/14.json"
fetch "/user_actions.json?username=dl-proto&filter=1&offset=0"
# New API loading style.
fetch "/t/26.json"
fetch "/t/26.json?page=2"