diff --git a/src/plugins/discourse/__snapshots__/fetch.test.js.snap b/src/plugins/discourse/__snapshots__/fetch.test.js.snap index 90a8e5d..d72aab0 100644 --- a/src/plugins/discourse/__snapshots__/fetch.test.js.snap +++ b/src/plugins/discourse/__snapshots__/fetch.test.js.snap @@ -455,6 +455,14 @@ The first paragraph of this pinned topic will be visible as a welcome message to ] `; +exports[`plugins/discourse/fetch snapshot testing loads topic IDs that are category definition topics 1`] = ` +Set { + 24, + 1, + 25, +} +`; + exports[`plugins/discourse/fetch snapshot testing loads user likes from snapshot 1`] = ` Array [ Object { diff --git a/src/plugins/discourse/fetch.js b/src/plugins/discourse/fetch.js index f3ed982..597b7bb 100644 --- a/src/plugins/discourse/fetch.js +++ b/src/plugins/discourse/fetch.js @@ -108,6 +108,10 @@ export interface Discourse { targetUsername: string, offset: number ): Promise; + + // Gets the topic IDs for every "about-x-category" topic. + // Discourse calls this a "definition" topic. + categoryDefinitionTopicIds(): Promise>; } const MAX_API_REQUESTS_PER_MINUTE = 55; @@ -164,6 +168,57 @@ export class Fetcher implements Discourse { return this._fetchImplementation(fullUrl, fetchOptions); } + async categoryDefinitionTopicIds(): Promise> { + const topicIdRE = new RegExp("/t/[\\w-]+/(\\d+)$"); + const urls: string[] = []; + const categoriesWithSubcategories: CategoryId[] = []; + + // Root categories + const response = await this._fetch( + `/categories.json?show_subcategory_list=true` + ); + failIfMissing(response); + failForNotOk(response); + const {categories: rootCategories} = (await response.json()).category_list; + for (const cat of rootCategories) { + if (cat.topic_url != null) { + urls.push(cat.topic_url); + } + if (cat.subcategory_ids) { + categoriesWithSubcategories.push(cat.id); + } + } + + // Subcategories + for (const rootCatId of categoriesWithSubcategories) { + const subResponse = await this._fetch( + `/categories.json?show_subcategory_list=true&parent_category_id=${rootCatId}` + ); + failIfMissing(subResponse); + failForNotOk(subResponse); + const {categories: subCategories} = ( + await subResponse.json() + ).category_list; + for (const cat of subCategories) { + if (cat.topic_url != null) { + urls.push(cat.topic_url); + } + } + } + + const ids = urls.map((url) => { + const match = topicIdRE.exec(url); + if (match == null) { + throw new Error( + `Encountered topic URL we failed to parse it's TopicId from: ${url}` + ); + } + return Number(match[1]); + }); + + return new Set(ids); + } + async latestTopicId(): Promise { const response = await this._fetch("/latest.json?order=created"); failIfMissing(response); @@ -172,6 +227,7 @@ export class Fetcher implements Discourse { if (json.topic_list.topics.length === 0) { throw new Error(`no topics! got ${stringify(json)} as latest topics.`); } + return json.topic_list.topics[0].id; } diff --git a/src/plugins/discourse/fetch.test.js b/src/plugins/discourse/fetch.test.js index 251ff09..51d7563 100644 --- a/src/plugins/discourse/fetch.test.js +++ b/src/plugins/discourse/fetch.test.js @@ -26,6 +26,11 @@ describe("plugins/discourse/fetch", () => { await snapshotFetcher().likesByUser("dl-proto", 0) ).toMatchSnapshot(); }); + it("loads topic IDs that are category definition topics", async () => { + expect( + await snapshotFetcher().categoryDefinitionTopicIds() + ).toMatchSnapshot(); + }); }); describe("error handling", () => { @@ -53,6 +58,11 @@ describe("plugins/discourse/fetch", () => { expectError("topic", (x) => x.topicWithPosts(14), 429); expectError("post", (x) => x.post(14), 429); + expectError( + "categoryDefinitionTopicIds", + (x) => x.categoryDefinitionTopicIds(), + 429 + ); function expectNull(name, f, status) { it(`${name} returns null on ${String(status)}`, async () => { diff --git a/src/plugins/discourse/mirror.test.js b/src/plugins/discourse/mirror.test.js index 314253c..fe3985b 100644 --- a/src/plugins/discourse/mirror.test.js +++ b/src/plugins/discourse/mirror.test.js @@ -41,6 +41,10 @@ class MockFetcher implements Discourse { this._likes = []; } + async categoryDefinitionTopicIds(): Promise> { + throw new Error("Method should not be used yet by mirror"); + } + async latestTopicId(): Promise { return this._latestTopicId; } diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZQ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZQ new file mode 100644 index 0000000..2d67451 --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZQ @@ -0,0 +1,120 @@ +{ + "category_list": { + "can_create_category": false, + "can_create_topic": false, + "draft": null, + "draft_key": "new_topic", + "draft_sequence": null, + "categories": [ + { + "id": 5, + "name": "Filler", + "color": "0088CC", + "text_color": "FFFFFF", + "slug": "filler", + "topic_count": 31, + "post_count": 0, + "position": 4, + "description": "This category fills up the forum with enough content to paginate. No important contents are added.", + "description_text": "This category fills up the forum with enough content to paginate. No important contents are added.", + "topic_url": "/t/about-the-filler-category/24", + "read_restricted": false, + "permission": null, + "notification_level": null, + "topic_template": null, + "has_children": true, + "sort_order": null, + "sort_ascending": null, + "show_subcategory_list": false, + "num_featured_topics": 3, + "default_view": null, + "subcategory_list_style": "rows_with_featured_topics", + "default_top_period": "all", + "minimum_required_tags": 0, + "navigate_to_first_post_after_read": false, + "topics_day": 0, + "topics_week": 0, + "topics_month": 0, + "topics_year": 0, + "topics_all_time": 32, + "description_excerpt": "This category fills up the forum with enough content to paginate. No important contents are added.", + "subcategory_ids": [ + 6 + ], + "uploaded_logo": null, + "uploaded_background": null + }, + { + "id": 1, + "name": "Uncategorized", + "color": "0088CC", + "text_color": "FFFFFF", + "slug": "uncategorized", + "topic_count": 7, + "post_count": 11, + "position": 0, + "description": "Topics that don't need a category, or don't fit into any other existing category.", + "description_text": "", + "topic_url": null, + "read_restricted": false, + "permission": null, + "notification_level": null, + "topic_template": null, + "has_children": false, + "sort_order": null, + "sort_ascending": null, + "show_subcategory_list": false, + "num_featured_topics": 3, + "default_view": null, + "subcategory_list_style": "rows_with_featured_topics", + "default_top_period": "all", + "minimum_required_tags": 0, + "navigate_to_first_post_after_read": false, + "topics_day": 0, + "topics_week": 0, + "topics_month": 1, + "topics_year": 7, + "topics_all_time": 7, + "description_excerpt": "Topics that don't need a category, or don't fit into any other existing category.", + "is_uncategorized": true, + "uploaded_logo": null, + "uploaded_background": null + }, + { + "id": 2, + "name": "Site Feedback", + "color": "808281", + "text_color": "FFFFFF", + "slug": "site-feedback", + "topic_count": 0, + "post_count": 0, + "position": 1, + "description": "Discussion about this site, its organization, how it works, and how we can improve it.", + "description_text": "Discussion about this site, its organization, how it works, and how we can improve it.", + "topic_url": "/t/about-the-site-feedback-category/1", + "read_restricted": false, + "permission": null, + "notification_level": null, + "topic_template": null, + "has_children": false, + "sort_order": null, + "sort_ascending": null, + "show_subcategory_list": false, + "num_featured_topics": 3, + "default_view": null, + "subcategory_list_style": "rows_with_featured_topics", + "default_top_period": "all", + "minimum_required_tags": 0, + "navigate_to_first_post_after_read": false, + "topics_day": 0, + "topics_week": 0, + "topics_month": 0, + "topics_year": 0, + "topics_all_time": 0, + "description_excerpt": "Discussion about this site, its organization, how it works, and how we can improve it.", + "uploaded_logo": null, + "uploaded_background": null + } + ] + } +} diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZSZwYXJlbnRfY2F0ZWdvcnlfaWQ9NQ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZSZwYXJlbnRfY2F0ZWdvcnlfaWQ9NQ new file mode 100644 index 0000000..80b4081 --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZSZwYXJlbnRfY2F0ZWdvcnlfaWQ9NQ @@ -0,0 +1,78 @@ +{ + "category_list": { + "can_create_category": false, + "can_create_topic": false, + "draft": null, + "draft_key": "new_topic", + "draft_sequence": null, + "categories": [ + { + "id": 6, + "name": "Big Filler", + "color": "9EB83B", + "text_color": "FFFFFF", + "slug": "big-filler", + "topic_count": 1, + "post_count": 0, + "position": 5, + "description": "Like Filler, but topics have enough posts to paginate.", + "description_text": "Like Filler, but topics have enough posts to paginate.", + "topic_url": "/t/about-the-big-filler-category/25", + "read_restricted": false, + "permission": null, + "parent_category_id": 5, + "notification_level": null, + "topic_template": null, + "has_children": false, + "sort_order": null, + "sort_ascending": null, + "show_subcategory_list": false, + "num_featured_topics": 3, + "default_view": null, + "subcategory_list_style": "rows_with_featured_topics", + "default_top_period": "all", + "minimum_required_tags": 0, + "navigate_to_first_post_after_read": false, + "topics_day": 0, + "topics_week": 0, + "topics_month": 0, + "topics_year": 0, + "topics_all_time": 1, + "description_excerpt": "Like Filler, but topics have enough posts to paginate.", + "uploaded_logo": null, + "uploaded_background": null, + "topics": [ + { + "id": 26, + "title": "Big Filler Topic 1", + "fancy_title": "Big Filler Topic 1", + "slug": "big-filler-topic-1", + "posts_count": 21, + "reply_count": 0, + "highest_post_number": 21, + "image_url": null, + "created_at": "2019-11-15T17:42:13.506Z", + "last_posted_at": "2019-11-15T17:44:32.244Z", + "bumped": true, + "bumped_at": "2019-11-15T17:44:32.244Z", + "unseen": false, + "pinned": false, + "unpinned": null, + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "has_accepted_answer": false, + "last_poster": { + "id": 5, + "username": "beanow.sc-test", + "name": "Beanow", + "avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png" + } + } + ] + } + ] + } +} diff --git a/src/plugins/discourse/update_discourse_api_snapshots.sh b/src/plugins/discourse/update_discourse_api_snapshots.sh index 8bcd4f6..aafa0cb 100755 --- a/src/plugins/discourse/update_discourse_api_snapshots.sh +++ b/src/plugins/discourse/update_discourse_api_snapshots.sh @@ -4,7 +4,6 @@ set -eu snapshots_dir=src/plugins/discourse/snapshots test_instance_url="https://sourcecred-test.discourse.group" -test_instance_username="credbot" if [ ! "$(jq --version)" ]; then printf >&2 'This script depends on jq. Please install it.\n' @@ -33,5 +32,7 @@ fetch "/posts/14.json" fetch "/user_actions.json?username=dl-proto&filter=1&offset=0" # New API loading style. +fetch "/categories.json?show_subcategory_list=true" +fetch "/categories.json?show_subcategory_list=true&parent_category_id=5" fetch "/t/26.json" fetch "/t/26.json?page=2"