mirror of
https://github.com/status-im/sourcecred.git
synced 2025-01-31 23:05:36 +00:00
Discourse: add categoryDefinitionTopicIds to fetcher (#1456)
This commit is contained in:
Robin van Boven
GitHub
parent
623c362246
commit
51e3eb8c25
@ -455,6 +455,14 @@ The first paragraph of this pinned topic will be visible as a welcome message to
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`plugins/discourse/fetch snapshot testing loads topic IDs that are category definition topics 1`] = `
|
||||
Set {
|
||||
24,
|
||||
1,
|
||||
25,
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`plugins/discourse/fetch snapshot testing loads user likes from snapshot 1`] = `
|
||||
Array [
|
||||
Object {
|
||||
|
||||
@ -108,6 +108,10 @@ export interface Discourse {
|
||||
targetUsername: string,
|
||||
offset: number
|
||||
): Promise<LikeAction[] | null>;
|
||||
|
||||
// Gets the topic IDs for every "about-x-category" topic.
|
||||
// Discourse calls this a "definition" topic.
|
||||
categoryDefinitionTopicIds(): Promise<Set<TopicId>>;
|
||||
}
|
||||
|
||||
const MAX_API_REQUESTS_PER_MINUTE = 55;
|
||||
@ -164,6 +168,57 @@ export class Fetcher implements Discourse {
|
||||
return this._fetchImplementation(fullUrl, fetchOptions);
|
||||
}
|
||||
|
||||
async categoryDefinitionTopicIds(): Promise<Set<TopicId>> {
|
||||
const topicIdRE = new RegExp("/t/[\\w-]+/(\\d+)$");
|
||||
const urls: string[] = [];
|
||||
const categoriesWithSubcategories: CategoryId[] = [];
|
||||
|
||||
// Root categories
|
||||
const response = await this._fetch(
|
||||
`/categories.json?show_subcategory_list=true`
|
||||
);
|
||||
failIfMissing(response);
|
||||
failForNotOk(response);
|
||||
const {categories: rootCategories} = (await response.json()).category_list;
|
||||
for (const cat of rootCategories) {
|
||||
if (cat.topic_url != null) {
|
||||
urls.push(cat.topic_url);
|
||||
}
|
||||
if (cat.subcategory_ids) {
|
||||
categoriesWithSubcategories.push(cat.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Subcategories
|
||||
for (const rootCatId of categoriesWithSubcategories) {
|
||||
const subResponse = await this._fetch(
|
||||
`/categories.json?show_subcategory_list=true&parent_category_id=${rootCatId}`
|
||||
);
|
||||
failIfMissing(subResponse);
|
||||
failForNotOk(subResponse);
|
||||
const {categories: subCategories} = (
|
||||
await subResponse.json()
|
||||
).category_list;
|
||||
for (const cat of subCategories) {
|
||||
if (cat.topic_url != null) {
|
||||
urls.push(cat.topic_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ids = urls.map((url) => {
|
||||
const match = topicIdRE.exec(url);
|
||||
if (match == null) {
|
||||
throw new Error(
|
||||
`Encountered topic URL we failed to parse it's TopicId from: ${url}`
|
||||
);
|
||||
}
|
||||
return Number(match[1]);
|
||||
});
|
||||
|
||||
return new Set(ids);
|
||||
}
|
||||
|
||||
async latestTopicId(): Promise<TopicId> {
|
||||
const response = await this._fetch("/latest.json?order=created");
|
||||
failIfMissing(response);
|
||||
@ -172,6 +227,7 @@ export class Fetcher implements Discourse {
|
||||
if (json.topic_list.topics.length === 0) {
|
||||
throw new Error(`no topics! got ${stringify(json)} as latest topics.`);
|
||||
}
|
||||
|
||||
return json.topic_list.topics[0].id;
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,11 @@ describe("plugins/discourse/fetch", () => {
|
||||
await snapshotFetcher().likesByUser("dl-proto", 0)
|
||||
).toMatchSnapshot();
|
||||
});
|
||||
it("loads topic IDs that are category definition topics", async () => {
|
||||
expect(
|
||||
await snapshotFetcher().categoryDefinitionTopicIds()
|
||||
).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe("error handling", () => {
|
||||
@ -53,6 +58,11 @@ describe("plugins/discourse/fetch", () => {
|
||||
|
||||
expectError("topic", (x) => x.topicWithPosts(14), 429);
|
||||
expectError("post", (x) => x.post(14), 429);
|
||||
expectError(
|
||||
"categoryDefinitionTopicIds",
|
||||
(x) => x.categoryDefinitionTopicIds(),
|
||||
429
|
||||
);
|
||||
|
||||
function expectNull(name, f, status) {
|
||||
it(`${name} returns null on ${String(status)}`, async () => {
|
||||
|
||||
@ -41,6 +41,10 @@ class MockFetcher implements Discourse {
|
||||
this._likes = [];
|
||||
}
|
||||
|
||||
async categoryDefinitionTopicIds(): Promise<Set<TopicId>> {
|
||||
throw new Error("Method should not be used yet by mirror");
|
||||
}
|
||||
|
||||
async latestTopicId(): Promise<TopicId> {
|
||||
return this._latestTopicId;
|
||||
}
|
||||
|
||||
@ -0,0 +1,120 @@
|
||||
{
|
||||
"category_list": {
|
||||
"can_create_category": false,
|
||||
"can_create_topic": false,
|
||||
"draft": null,
|
||||
"draft_key": "new_topic",
|
||||
"draft_sequence": null,
|
||||
"categories": [
|
||||
{
|
||||
"id": 5,
|
||||
"name": "Filler",
|
||||
"color": "0088CC",
|
||||
"text_color": "FFFFFF",
|
||||
"slug": "filler",
|
||||
"topic_count": 31,
|
||||
"post_count": 0,
|
||||
"position": 4,
|
||||
"description": "This category fills up the forum with enough content to paginate. No important contents are added.",
|
||||
"description_text": "This category fills up the forum with enough content to paginate. No important contents are added.",
|
||||
"topic_url": "/t/about-the-filler-category/24",
|
||||
"read_restricted": false,
|
||||
"permission": null,
|
||||
"notification_level": null,
|
||||
"topic_template": null,
|
||||
"has_children": true,
|
||||
"sort_order": null,
|
||||
"sort_ascending": null,
|
||||
"show_subcategory_list": false,
|
||||
"num_featured_topics": 3,
|
||||
"default_view": null,
|
||||
"subcategory_list_style": "rows_with_featured_topics",
|
||||
"default_top_period": "all",
|
||||
"minimum_required_tags": 0,
|
||||
"navigate_to_first_post_after_read": false,
|
||||
"topics_day": 0,
|
||||
"topics_week": 0,
|
||||
"topics_month": 0,
|
||||
"topics_year": 0,
|
||||
"topics_all_time": 32,
|
||||
"description_excerpt": "This category fills up the forum with enough content to paginate. No important contents are added.",
|
||||
"subcategory_ids": [
|
||||
6
|
||||
],
|
||||
"uploaded_logo": null,
|
||||
"uploaded_background": null
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Uncategorized",
|
||||
"color": "0088CC",
|
||||
"text_color": "FFFFFF",
|
||||
"slug": "uncategorized",
|
||||
"topic_count": 7,
|
||||
"post_count": 11,
|
||||
"position": 0,
|
||||
"description": "Topics that don't need a category, or don't fit into any other existing category.",
|
||||
"description_text": "",
|
||||
"topic_url": null,
|
||||
"read_restricted": false,
|
||||
"permission": null,
|
||||
"notification_level": null,
|
||||
"topic_template": null,
|
||||
"has_children": false,
|
||||
"sort_order": null,
|
||||
"sort_ascending": null,
|
||||
"show_subcategory_list": false,
|
||||
"num_featured_topics": 3,
|
||||
"default_view": null,
|
||||
"subcategory_list_style": "rows_with_featured_topics",
|
||||
"default_top_period": "all",
|
||||
"minimum_required_tags": 0,
|
||||
"navigate_to_first_post_after_read": false,
|
||||
"topics_day": 0,
|
||||
"topics_week": 0,
|
||||
"topics_month": 1,
|
||||
"topics_year": 7,
|
||||
"topics_all_time": 7,
|
||||
"description_excerpt": "Topics that don't need a category, or don't fit into any other existing category.",
|
||||
"is_uncategorized": true,
|
||||
"uploaded_logo": null,
|
||||
"uploaded_background": null
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Site Feedback",
|
||||
"color": "808281",
|
||||
"text_color": "FFFFFF",
|
||||
"slug": "site-feedback",
|
||||
"topic_count": 0,
|
||||
"post_count": 0,
|
||||
"position": 1,
|
||||
"description": "Discussion about this site, its organization, how it works, and how we can improve it.",
|
||||
"description_text": "Discussion about this site, its organization, how it works, and how we can improve it.",
|
||||
"topic_url": "/t/about-the-site-feedback-category/1",
|
||||
"read_restricted": false,
|
||||
"permission": null,
|
||||
"notification_level": null,
|
||||
"topic_template": null,
|
||||
"has_children": false,
|
||||
"sort_order": null,
|
||||
"sort_ascending": null,
|
||||
"show_subcategory_list": false,
|
||||
"num_featured_topics": 3,
|
||||
"default_view": null,
|
||||
"subcategory_list_style": "rows_with_featured_topics",
|
||||
"default_top_period": "all",
|
||||
"minimum_required_tags": 0,
|
||||
"navigate_to_first_post_after_read": false,
|
||||
"topics_day": 0,
|
||||
"topics_week": 0,
|
||||
"topics_month": 0,
|
||||
"topics_year": 0,
|
||||
"topics_all_time": 0,
|
||||
"description_excerpt": "Discussion about this site, its organization, how it works, and how we can improve it.",
|
||||
"uploaded_logo": null,
|
||||
"uploaded_background": null
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
{
|
||||
"category_list": {
|
||||
"can_create_category": false,
|
||||
"can_create_topic": false,
|
||||
"draft": null,
|
||||
"draft_key": "new_topic",
|
||||
"draft_sequence": null,
|
||||
"categories": [
|
||||
{
|
||||
"id": 6,
|
||||
"name": "Big Filler",
|
||||
"color": "9EB83B",
|
||||
"text_color": "FFFFFF",
|
||||
"slug": "big-filler",
|
||||
"topic_count": 1,
|
||||
"post_count": 0,
|
||||
"position": 5,
|
||||
"description": "Like Filler, but topics have enough posts to paginate.",
|
||||
"description_text": "Like Filler, but topics have enough posts to paginate.",
|
||||
"topic_url": "/t/about-the-big-filler-category/25",
|
||||
"read_restricted": false,
|
||||
"permission": null,
|
||||
"parent_category_id": 5,
|
||||
"notification_level": null,
|
||||
"topic_template": null,
|
||||
"has_children": false,
|
||||
"sort_order": null,
|
||||
"sort_ascending": null,
|
||||
"show_subcategory_list": false,
|
||||
"num_featured_topics": 3,
|
||||
"default_view": null,
|
||||
"subcategory_list_style": "rows_with_featured_topics",
|
||||
"default_top_period": "all",
|
||||
"minimum_required_tags": 0,
|
||||
"navigate_to_first_post_after_read": false,
|
||||
"topics_day": 0,
|
||||
"topics_week": 0,
|
||||
"topics_month": 0,
|
||||
"topics_year": 0,
|
||||
"topics_all_time": 1,
|
||||
"description_excerpt": "Like Filler, but topics have enough posts to paginate.",
|
||||
"uploaded_logo": null,
|
||||
"uploaded_background": null,
|
||||
"topics": [
|
||||
{
|
||||
"id": 26,
|
||||
"title": "Big Filler Topic 1",
|
||||
"fancy_title": "Big Filler Topic 1",
|
||||
"slug": "big-filler-topic-1",
|
||||
"posts_count": 21,
|
||||
"reply_count": 0,
|
||||
"highest_post_number": 21,
|
||||
"image_url": null,
|
||||
"created_at": "2019-11-15T17:42:13.506Z",
|
||||
"last_posted_at": "2019-11-15T17:44:32.244Z",
|
||||
"bumped": true,
|
||||
"bumped_at": "2019-11-15T17:44:32.244Z",
|
||||
"unseen": false,
|
||||
"pinned": false,
|
||||
"unpinned": null,
|
||||
"visible": true,
|
||||
"closed": false,
|
||||
"archived": false,
|
||||
"bookmarked": null,
|
||||
"liked": null,
|
||||
"has_accepted_answer": false,
|
||||
"last_poster": {
|
||||
"id": 5,
|
||||
"username": "beanow.sc-test",
|
||||
"name": "Beanow",
|
||||
"avatar_template": "https://avatars.discourse.org/v4/letter/b/7ea924/{size}.png"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -4,7 +4,6 @@ set -eu
|
||||
|
||||
snapshots_dir=src/plugins/discourse/snapshots
|
||||
test_instance_url="https://sourcecred-test.discourse.group"
|
||||
test_instance_username="credbot"
|
||||
|
||||
if [ ! "$(jq --version)" ]; then
|
||||
printf >&2 'This script depends on jq. Please install it.\n'
|
||||
@ -33,5 +32,7 @@ fetch "/posts/14.json"
|
||||
fetch "/user_actions.json?username=dl-proto&filter=1&offset=0"
|
||||
|
||||
# New API loading style.
|
||||
fetch "/categories.json?show_subcategory_list=true"
|
||||
fetch "/categories.json?show_subcategory_list=true&parent_category_id=5"
|
||||
fetch "/t/26.json"
|
||||
fetch "/t/26.json?page=2"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user