Discourse: add categoryDefinitionTopicIds to fetcher (#1456)

src/plugins/discourse/__snapshots__/fetch.test.js.snap

@@ -455,6 +455,14 @@ The first paragraph of this pinned topic will be visible as a welcome message to
 ]
 `;
 
+exports[`plugins/discourse/fetch snapshot testing loads topic IDs that are category definition topics 1`] = `
+Set {
+  24,
+  1,
+  25,
+}
+`;
+
 exports[`plugins/discourse/fetch snapshot testing loads user likes from snapshot 1`] = `
 Array [
   Object {

src/plugins/discourse/fetch.js

@@ -108,6 +108,10 @@ export interface Discourse {
     targetUsername: string,
     offset: number
   ): Promise<LikeAction[] | null>;
+
+  // Gets the topic IDs for every "about-x-category" topic.
+  // Discourse calls this a "definition" topic.
+  categoryDefinitionTopicIds(): Promise<Set<TopicId>>;
 }
 
 const MAX_API_REQUESTS_PER_MINUTE = 55;
@@ -164,6 +168,57 @@ export class Fetcher implements Discourse {
     return this._fetchImplementation(fullUrl, fetchOptions);
   }
 
+  async categoryDefinitionTopicIds(): Promise<Set<TopicId>> {
+    const topicIdRE = new RegExp("/t/[\\w-]+/(\\d+)$");
+    const urls: string[] = [];
+    const categoriesWithSubcategories: CategoryId[] = [];
+
+    // Root categories
+    const response = await this._fetch(
+      `/categories.json?show_subcategory_list=true`
+    );
+    failIfMissing(response);
+    failForNotOk(response);
+    const {categories: rootCategories} = (await response.json()).category_list;
+    for (const cat of rootCategories) {
+      if (cat.topic_url != null) {
+        urls.push(cat.topic_url);
+      }
+      if (cat.subcategory_ids) {
+        categoriesWithSubcategories.push(cat.id);
+      }
+    }
+
+    // Subcategories
+    for (const rootCatId of categoriesWithSubcategories) {
+      const subResponse = await this._fetch(
+        `/categories.json?show_subcategory_list=true&parent_category_id=${rootCatId}`
+      );
+      failIfMissing(subResponse);
+      failForNotOk(subResponse);
+      const {categories: subCategories} = (
+        await subResponse.json()
+      ).category_list;
+      for (const cat of subCategories) {
+        if (cat.topic_url != null) {
+          urls.push(cat.topic_url);
+        }
+      }
+    }
+
+    const ids = urls.map((url) => {
+      const match = topicIdRE.exec(url);
+      if (match == null) {
+        throw new Error(
+          `Encountered topic URL we failed to parse it's TopicId from: ${url}`
+        );
+      }
+      return Number(match[1]);
+    });
+
+    return new Set(ids);
+  }
+
   async latestTopicId(): Promise<TopicId> {
     const response = await this._fetch("/latest.json?order=created");
     failIfMissing(response);
@@ -172,6 +227,7 @@ export class Fetcher implements Discourse {
     if (json.topic_list.topics.length === 0) {
       throw new Error(`no topics! got ${stringify(json)} as latest topics.`);
     }
+
     return json.topic_list.topics[0].id;
   }
 

src/plugins/discourse/fetch.test.js

@@ -26,6 +26,11 @@ describe("plugins/discourse/fetch", () => {
         await snapshotFetcher().likesByUser("dl-proto", 0)
       ).toMatchSnapshot();
     });
+    it("loads topic IDs that are category definition topics", async () => {
+      expect(
+        await snapshotFetcher().categoryDefinitionTopicIds()
+      ).toMatchSnapshot();
+    });
   });
 
   describe("error handling", () => {
@@ -53,6 +58,11 @@ describe("plugins/discourse/fetch", () => {
 
     expectError("topic", (x) => x.topicWithPosts(14), 429);
     expectError("post", (x) => x.post(14), 429);
+    expectError(
+      "categoryDefinitionTopicIds",
+      (x) => x.categoryDefinitionTopicIds(),
+      429
+    );
 
     function expectNull(name, f, status) {
       it(`${name} returns null on ${String(status)}`, async () => {

src/plugins/discourse/mirror.test.js

@@ -41,6 +41,10 @@ class MockFetcher implements Discourse {
     this._likes = [];
   }
 
+  async categoryDefinitionTopicIds(): Promise<Set<TopicId>> {
+    throw new Error("Method should not be used yet by mirror");
+  }
+
   async latestTopicId(): Promise<TopicId> {
     return this._latestTopicId;
   }

src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZQ

@@ -0,0 +1,120 @@
+{
+  "category_list": {
+    "can_create_category": false,
+    "can_create_topic": false,
+    "draft": null,
+    "draft_key": "new_topic",
+    "draft_sequence": null,
+    "categories": [
+      {
+        "id": 5,
+        "name": "Filler",
+        "color": "0088CC",
+        "text_color": "FFFFFF",
+        "slug": "filler",
+        "topic_count": 31,
+        "post_count": 0,
+        "position": 4,
+        "description": "This category fills up the forum with enough content to paginate. No important contents are added.",
+        "description_text": "This category fills up the forum with enough content to paginate. No important contents are added.",
+        "topic_url": "/t/about-the-filler-category/24",
+        "read_restricted": false,
+        "permission": null,
+        "notification_level": null,
+        "topic_template": null,
+        "has_children": true,
+        "sort_order": null,
+        "sort_ascending": null,
+        "show_subcategory_list": false,
+        "num_featured_topics": 3,
+        "default_view": null,
+        "subcategory_list_style": "rows_with_featured_topics",
+        "default_top_period": "all",
+        "minimum_required_tags": 0,
+        "navigate_to_first_post_after_read": false,
+        "topics_day": 0,
+        "topics_week": 0,
+        "topics_month": 0,
+        "topics_year": 0,
+        "topics_all_time": 32,
+        "description_excerpt": "This category fills up the forum with enough content to paginate. No important contents are added.",
+        "subcategory_ids": [
+          6
+        ],
+        "uploaded_logo": null,
+        "uploaded_background": null
+      },
+      {
+        "id": 1,
+        "name": "Uncategorized",
+        "color": "0088CC",
+        "text_color": "FFFFFF",
+        "slug": "uncategorized",
+        "topic_count": 7,
+        "post_count": 11,
+        "position": 0,
+        "description": "Topics that don't need a category, or don't fit into any other existing category.",
+        "description_text": "",
+        "topic_url": null,
+        "read_restricted": false,
+        "permission": null,
+        "notification_level": null,
+        "topic_template": null,
+        "has_children": false,
+        "sort_order": null,
+        "sort_ascending": null,
+        "show_subcategory_list": false,
+        "num_featured_topics": 3,
+        "default_view": null,
+        "subcategory_list_style": "rows_with_featured_topics",
+        "default_top_period": "all",
+        "minimum_required_tags": 0,
+        "navigate_to_first_post_after_read": false,
+        "topics_day": 0,
+        "topics_week": 0,
+        "topics_month": 1,
+        "topics_year": 7,
+        "topics_all_time": 7,
+        "description_excerpt": "Topics that don't need a category, or don't fit into any other existing category.",
+        "is_uncategorized": true,
+        "uploaded_logo": null,
+        "uploaded_background": null
+      },
+      {
+        "id": 2,
+        "name": "Site Feedback",
+        "color": "808281",
+        "text_color": "FFFFFF",
+        "slug": "site-feedback",
+        "topic_count": 0,
+        "post_count": 0,
+        "position": 1,
+        "description": "Discussion about this site, its organization, how it works, and how we can improve it.",
+        "description_text": "Discussion about this site, its organization, how it works, and how we can improve it.",
+        "topic_url": "/t/about-the-site-feedback-category/1",
+        "read_restricted": false,
+        "permission": null,
+        "notification_level": null,
+        "topic_template": null,
+        "has_children": false,
+        "sort_order": null,
+        "sort_ascending": null,
+        "show_subcategory_list": false,
+        "num_featured_topics": 3,
+        "default_view": null,
+        "subcategory_list_style": "rows_with_featured_topics",
+        "default_top_period": "all",
+        "minimum_required_tags": 0,
+        "navigate_to_first_post_after_read": false,
+        "topics_day": 0,
+        "topics_week": 0,
+        "topics_month": 0,
+        "topics_year": 0,
+        "topics_all_time": 0,
+        "description_excerpt": "Discussion about this site, its organization, how it works, and how we can improve it.",
+        "uploaded_logo": null,
+        "uploaded_background": null
+      }
+    ]
+  }
+}

src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2NhdGVnb3JpZXMuanNvbj9zaG93X3N1YmNhdGVnb3J5X2xpc3Q9dHJ1ZSZwYXJlbnRfY2F0ZWdvcnlfaWQ9NQ

@@ -0,0 +1,78 @@
+{
+  "category_list": {
+    "can_create_category": false,
+    "can_create_topic": false,
+    "draft": null,
+    "draft_key": "new_topic",
+    "draft_sequence": null,
+    "categories": [
+      {
+        "id": 6,
+        "name": "Big Filler",
+        "color": "9EB83B",
+        "text_color": "FFFFFF",
+        "slug": "big-filler",
+        "topic_count": 1,
+        "post_count": 0,
+        "position": 5,
+        "description": "Like Filler, but topics have enough posts to paginate.",
+        "description_text": "Like Filler, but topics have enough posts to paginate.",
+        "topic_url": "/t/about-the-big-filler-category/25",
+        "read_restricted": false,
+        "permission": null,
+        "parent_category_id": 5,
+        "notification_level": null,
+        "topic_template": null,
+        "has_children": false,
+        "sort_order": null,
+        "sort_ascending": null,
+        "show_subcategory_list": false,
+        "num_featured_topics": 3,
+        "default_view": null,
+        "subcategory_list_style": "rows_with_featured_topics",
+        "default_top_period": "all",
+        "minimum_required_tags": 0,
+        "navigate_to_first_post_after_read": false,
+        "topics_day": 0,
+        "topics_week": 0,
+        "topics_month": 0,
+        "topics_year": 0,
+        "topics_all_time": 1,
+        "description_excerpt": "Like Filler, but topics have enough posts to paginate.",
+        "uploaded_logo": null,
+        "uploaded_background": null,
+        "topics": [
+          {
+            "id": 26,
+            "title": "Big Filler Topic 1",
+            "fancy_title": "Big Filler Topic 1",
+            "slug": "big-filler-topic-1",
+            "posts_count": 21,
+            "reply_count": 0,
+            "highest_post_number": 21,
+            "image_url": null,
+            "created_at": "2019-11-15T17:42:13.506Z",
+            "last_posted_at": "2019-11-15T17:44:32.244Z",
+            "bumped": true,
+            "bumped_at": "2019-11-15T17:44:32.244Z",
+            "unseen": false,
+            "pinned": false,
+            "unpinned": null,
+            "visible": true,
+            "closed": false,
+            "archived": false,
+            "bookmarked": null,
+            "liked": null,
+            "has_accepted_answer": false,
+            "last_poster": {
+              "id": 5,
+              "username": "beanow.sc-test",
+              "name": "Beanow",
+              "avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
+            }
+          }
+        ]
+      }
+    ]
+  }
+}

src/plugins/discourse/update_discourse_api_snapshots.sh

@@ -4,7 +4,6 @@ set -eu
 
 snapshots_dir=src/plugins/discourse/snapshots
 test_instance_url="https://sourcecred-test.discourse.group"
-test_instance_username="credbot"
 
 if [ ! "$(jq --version)" ]; then
   printf >&2 'This script depends on jq. Please install it.\n'
@@ -33,5 +32,7 @@ fetch "/posts/14.json"
 fetch "/user_actions.json?username=dl-proto&filter=1&offset=0"
 
 # New API loading style.
+fetch "/categories.json?show_subcategory_list=true"
+fetch "/categories.json?show_subcategory_list=true&parent_category_id=5"
 fetch "/t/26.json"
 fetch "/t/26.json?page=2"