diff --git a/scripts/update_snapshots.sh b/scripts/update_snapshots.sh index 2ce02e6..ed1dc6f 100755 --- a/scripts/update_snapshots.sh +++ b/scripts/update_snapshots.sh @@ -34,5 +34,12 @@ echo "Updating github/fetchGithubOrgTest.sh" echo "Updating github/fetchGithubRepoTest.sh" ./src/plugins/github/fetchGithubRepoTest.sh -u --no-build +if [ -z "${DISCOURSE_TEST_API_KEY:-}" ]; then + echo "Updating Discourse API snapshots" + ./src/plugins/discourse/update_discourse_api_snapshots.sh +else + echo "Not updating Discourse API snapshots (need DISCOURSE_TEST_API_KEY)" +fi + echo "Updating Jest snapshots" yarn unit -u diff --git a/src/plugins/discourse/__snapshots__/fetch.test.js.snap b/src/plugins/discourse/__snapshots__/fetch.test.js.snap new file mode 100644 index 0000000..98ff41b --- /dev/null +++ b/src/plugins/discourse/__snapshots__/fetch.test.js.snap @@ -0,0 +1,110 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`plugins/discourse/fetch snapshot testing loads a particular post from snapshot 1`] = ` +Object { + "authorUsername": "d11", + "id": 14, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1564744349476, + "topicId": 11, +} +`; + +exports[`plugins/discourse/fetch snapshot testing loads a particular topic from snapshot 1`] = ` +Object { + "posts": Array [ + Object { + "authorUsername": "d11", + "id": 14, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1564744349476, + "topicId": 11, + }, + Object { + "authorUsername": "dl-proto", + "id": 16, + "indexWithinTopic": 2, + "replyToPostIndex": null, + "timestampMs": 1564744496285, + "topicId": 11, + }, + Object { + "authorUsername": "d11", + "id": 22, + "indexWithinTopic": 3, + "replyToPostIndex": null, + "timestampMs": 1564951839667, + "topicId": 11, + }, + ], + "topic": Object { + "authorUsername": "d11", + "id": 11, + "timestampMs": 1564744349408, + "title": "My First Test Post", + }, +} +`; + +exports[`plugins/discourse/fetch snapshot testing loads latest posts from snapshot 1`] = ` +Array [ + Object { + "authorUsername": "d11", + "id": 22, + "indexWithinTopic": 3, + "replyToPostIndex": null, + "timestampMs": 1564951839667, + "topicId": 11, + }, + Object { + "authorUsername": "d11", + "id": 18, + "indexWithinTopic": 2, + "replyToPostIndex": null, + "timestampMs": 1564744591313, + "topicId": 13, + }, + Object { + "authorUsername": "dl-proto", + "id": 17, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1564744553905, + "topicId": 13, + }, + Object { + "authorUsername": "dl-proto", + "id": 16, + "indexWithinTopic": 2, + "replyToPostIndex": null, + "timestampMs": 1564744496285, + "topicId": 11, + }, + Object { + "authorUsername": "d11", + "id": 14, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1564744349476, + "topicId": 11, + }, + Object { + "authorUsername": "system", + "id": 10, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1554236412933, + "topicId": 7, + }, + Object { + "authorUsername": "system", + "id": 1, + "indexWithinTopic": 1, + "replyToPostIndex": null, + "timestampMs": 1554236409844, + "topicId": 1, + }, +] +`; diff --git a/src/plugins/discourse/fetch.js b/src/plugins/discourse/fetch.js new file mode 100644 index 0000000..71c2367 --- /dev/null +++ b/src/plugins/discourse/fetch.js @@ -0,0 +1,218 @@ +// @flow + +/** + * Class for retrieving data from the Discourse API. + * + * The Discourse API implements the JSON endpoints for all functionality of the actual site. + * As such, it tends to return a lot of information that we don't care about (in contrast + * to a GraphQL API which would give us only what we ask for). As such, we implement a simple + * interface over it, which both abstracts over calling the API, and does some post-processing + * on the results to simplify it to data that is relevant for us. + */ + +import stringify from "json-stable-stringify"; +import fetch from "isomorphic-fetch"; + +export type UserId = number; +export type PostId = number; +export type TopicId = number; + +export type Topic = {| + +id: TopicId, + +title: string, + +timestampMs: number, + +authorUsername: string, +|}; + +export type Post = {| + +id: PostId, + +topicId: TopicId, + // Which number post this was within the topic (starts at 1) + +indexWithinTopic: number, + // The indexWithinTopic of the post within the same topic that this post was a + // reply to. Will be `null` if this post was the first post, or if it was a + // reply to the first post. + +replyToPostIndex: number | null, + +timestampMs: number, + +authorUsername: string, +|}; + +export type TopicWithPosts = {| + +topic: Topic, + // Not guaranteed to contain all the Posts in the topic—clients will need to + // manually request some posts. The raw response actually includes a list of + // all the PostIds in the topic, but for now we don't use them. + // + // We do use these Posts though, as it allows us to save requesting them all + // individually. + +posts: $ReadOnlyArray, +|}; + +/** + * Interface over the external Discourse API, structured to suit our particular needs. + * We have an interface (as opposed to just an implementation) to enable easy mocking and + * testing. + */ +export interface Discourse { + // Get the `id` of the latest topic on the server. + // Vital so that we can then enumerate and fetch every Topic we haven't seen yet. + // May reject on not OK status like 404 or 403. + latestTopicId(): Promise; + // Retrieve the Topic with Posts for a given id. + // Will resolve to null if the response status is 403 or 404. 403 because the + // topic may be hidden from the API user; 404 because we sometimes see + // 404s in prod and want to ignore those topic ids. (Not sure why it happens.) + // May reject if the status is not OK and is not 404 or 403. + topicWithPosts(id: TopicId): Promise; + // Retrieve an individual Post by its id. + // Will resolve to null if the response status is 403 or 404. 403 because the + // topic may be hidden from the API user; 404 because we sometimes see + // 404s in prod and want to ignore those topic ids. (Not sure why it happens.) + // May reject if the status is not OK and is not 404 or 403. + post(id: PostId): Promise; + // Retrieve the latest posts from the server. + // Vital so that we can then enumerate and fetch every Post that we haven't + // encountered. + // May reject on not OK status like 404 or 403. + latestPosts(): Promise; +} + +export class Fetcher implements Discourse { + +options: DiscourseFetchOptions; + +_fetchImplementation: typeof fetch; + + constructor( + options: DiscourseFetchOptions, + // fetchImplementation shouldn't be provided by clients, but is convenient for testing. + fetchImplementation?: typeof fetch + ) { + this.options = options; + this._fetchImplementation = fetchImplementation || fetch; + } + + _fetch(endpoint: string): Promise { + const {serverUrl, apiKey, apiUsername} = this.options; + if (!endpoint.startsWith("/")) { + throw new Error(`invalid endpoint: ${endpoint}`); + } + if (!serverUrl.startsWith("http") || serverUrl.endsWith("/")) { + throw new Error(`invalid server url: ${serverUrl}`); + } + const fetchOptions = { + method: "GET", + headers: { + "Api-Key": apiKey, + "Api-Username": apiUsername, + Accept: "application/json", + }, + }; + const fullUrl = `${serverUrl}${endpoint}`; + return this._fetchImplementation(fullUrl, fetchOptions); + } + + async latestTopicId(): Promise { + const response = await this._fetch("/latest.json?order=created"); + maybeFail404(response); + maybeFail403(response); + if (!response.ok) { + throw new Error(`not OK status ${response.status} on ${response.url}`); + } + const json = await response.json(); + if (json.topic_list.topics.length === 0) { + throw new Error(`no topics! got ${stringify(json)} as latest topics.`); + } + return json.topic_list.topics[0].id; + } + + async latestPosts(): Promise { + const response = await this._fetch("/posts.json"); + maybeFail404(response); + maybeFail403(response); + if (!response.ok) { + throw new Error(`not OK status ${response.status} on ${response.url}`); + } + const json = await response.json(); + return json.latest_posts.map(parsePost); + } + + async topicWithPosts(id: TopicId): Promise { + const response = await this._fetch(`/t/${id}.json`); + if (response.status === 404) { + // Not sure why this happens, but a topic can sometimes 404. + // We should just consider it unreachable. + // Here is an example: https://discourse.sourcecred.io/t/116 + return null; + } + if (response.status === 403) { + // Probably this topic is hidden or deleted. + // Just consider it unreachable. + // If the issue is that the user provided bad keys, then + // they will get a more helpful error when they try to get the latest + // topic id. + return null; + } + if (response.status !== 200) { + throw new Error(`not OK status ${response.status} on ${response.url}`); + } + const json = await response.json(); + const posts = json.post_stream.posts.map(parsePost); + const topic: Topic = { + id: json.id, + title: json.title, + timestampMs: +new Date(json.created_at), + authorUsername: json.details.created_by.username, + }; + return {topic, posts}; + } + + async post(id: PostId): Promise { + const response = await this._fetch(`/posts/${id}.json`); + if (response.status === 404) { + // Since topics can 404, I assume posts can too. + return null; + } + if (response.status === 403) { + // Probably this post is hidden or deleted. + return null; + } + if (response.status !== 200) { + throw new Error(`not OK status ${response.status} on ${response.url}`); + } + const json = await response.json(); + return parsePost(json); + } +} + +function maybeFail404(response) { + if (response.status === 404) { + throw new Error(`404 Not Found on: ${response.url}; maybe bad serverUrl?`); + } +} + +function maybeFail403(response) { + if (response.status === 403) { + throw new Error(`403 Forbidden: bad API username or key?`); + } +} + +function parsePost(json: any): Post { + return { + id: json.id, + timestampMs: Date.parse(json.created_at), + indexWithinTopic: json.post_number, + replyToPostIndex: json.reply_to_post_number, + topicId: json.topic_id, + authorUsername: json.username, + }; +} + +export type DiscourseFetchOptions = {| + apiKey: string, + // We'll use the view permissions for this user. It needs to be a real user + // on the server. I recommend making a new user called "credbot" with no + // special permissions for this purpose. If you use a permissioned user (e.g. + // "system") then SourceCred will pick up hidden and deleted posts, + // potentially leaking private information. + apiUsername: string, + serverUrl: string, +|}; diff --git a/src/plugins/discourse/fetch.test.js b/src/plugins/discourse/fetch.test.js new file mode 100644 index 0000000..ffc33c8 --- /dev/null +++ b/src/plugins/discourse/fetch.test.js @@ -0,0 +1,104 @@ +// @flow + +import deepFreeze from "deep-freeze"; +import {Fetcher, type DiscourseFetchOptions} from "./fetch"; +import base64url from "base64url"; +import path from "path"; +import fs from "fs-extra"; + +describe("plugins/discourse/fetch", () => { + const options: DiscourseFetchOptions = deepFreeze({ + apiKey: "FAKE_KEY", + apiUsername: "credbot", + serverUrl: "https://sourcecred-test.discourse.group", + }); + + describe("snapshot testing", () => { + async function snapshotFetch( + url: string | Request | URL + ): Promise { + const snapshotDir = "src/plugins/discourse/snapshots"; + const filename = base64url(url); + const file = path.join(snapshotDir, filename); + if (await fs.exists(file)) { + const contents = await fs.readFile(file); + return new Response(contents, {status: 200}); + } else { + throw new Error(`couldn't load snapshot for ${file}`); + } + } + const snapshotFetcher = () => new Fetcher(options, snapshotFetch); + + it("loads LatestTopicId from snapshot", async () => { + const topicId = await snapshotFetcher().latestTopicId(); + expect(topicId).toMatchInlineSnapshot(`13`); + }); + it("loads latest posts from snapshot", async () => { + expect(await snapshotFetcher().latestPosts()).toMatchSnapshot(); + }); + it("loads a particular topic from snapshot", async () => { + expect(await snapshotFetcher().topicWithPosts(11)).toMatchSnapshot(); + }); + it("loads a particular post from snapshot", async () => { + expect(await snapshotFetcher().post(14)).toMatchSnapshot(); + }); + }); + + describe("error handling", () => { + const fakeFetch = (status: number) => (url: any) => { + const resp = new Response("", {status, url}); + return Promise.resolve(resp); + }; + const fetcherWithStatus = (status: number) => + new Fetcher(options, fakeFetch(status)); + function expectError(name, f, status) { + it(`${name} errors on ${String(status)}`, () => { + const fetcher = fetcherWithStatus(status); + expect.assertions(1); + const result = f(fetcher); + return result.catch((e) => expect(e.message).toMatch(String(status))); + }); + } + expectError("latestTopicId", (x) => x.latestTopicId(), 404); + expectError("latestTopicId", (x) => x.latestTopicId(), 403); + expectError("latestTopicId", (x) => x.latestTopicId(), 429); + + expectError("latestPosts", (x) => x.latestPosts(), 404); + expectError("latestPosts", (x) => x.latestPosts(), 403); + expectError("latestPosts", (x) => x.latestPosts(), 429); + + expectError("topic", (x) => x.topicWithPosts(14), 429); + expectError("post", (x) => x.post(14), 429); + + function expectNull(name, f, status) { + it(`${name} returns null on ${String(status)}`, async () => { + const fetcher = fetcherWithStatus(status); + const result = f(fetcher); + expect(await result).toBe(null); + }); + } + + expectNull("topic", (x) => x.topicWithPosts(14), 404); + expectNull("topic", (x) => x.topicWithPosts(14), 403); + expectNull("post", (x) => x.post(14), 404); + expectNull("post", (x) => x.post(14), 403); + }); + + describe("fetch headers", () => { + it("calls fetch with the right options and headers", async () => { + let fetchOptions: ?RequestOptions; + const fakeFetch = (url, _options) => { + fetchOptions = _options; + return Promise.resolve(new Response("", {status: 404})); + }; + await new Fetcher(options, fakeFetch).post(1337); + if (fetchOptions == null) { + throw new Error("fetchOptions == null"); + } + expect(fetchOptions.method).toEqual("GET"); + expect(fetchOptions.headers["Api-Key"]).toEqual(options.apiKey); + expect(fetchOptions.headers["Api-Username"]).toEqual(options.apiUsername); + expect(fetchOptions.headers["Accept"]).toEqual("application/json"); + }); + }); +}); diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2xhdGVzdC5qc29uP29yZGVyPWNyZWF0ZWQ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2xhdGVzdC5qc29uP29yZGVyPWNyZWF0ZWQ new file mode 100644 index 0000000..08720f5 --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL2xhdGVzdC5qc29uP29yZGVyPWNyZWF0ZWQ @@ -0,0 +1,162 @@ +{ + "users": [ + { + "id": 3, + "username": "dl-proto", + "name": "dandelion protocol acct", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png" + }, + { + "id": 2, + "username": "d11", + "name": "D", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png" + }, + { + "id": -1, + "username": "system", + "name": "system", + "avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png" + } + ], + "primary_groups": [], + "topic_list": { + "can_create_topic": true, + "draft": null, + "draft_key": "new_topic", + "draft_sequence": 0, + "per_page": 30, + "topics": [ + { + "id": 13, + "title": "A Thread With Links", + "fancy_title": "A Thread With Links", + "slug": "a-thread-with-links", + "posts_count": 2, + "reply_count": 0, + "highest_post_number": 2, + "image_url": null, + "created_at": "2019-08-02T11:15:53.843Z", + "last_posted_at": "2019-08-02T11:16:31.313Z", + "bumped": true, + "bumped_at": "2019-08-02T11:17:12.755Z", + "unseen": false, + "pinned": false, + "unpinned": null, + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "views": 6, + "like_count": 0, + "has_summary": false, + "archetype": "regular", + "last_poster_username": "dl-proto", + "category_id": 1, + "pinned_globally": false, + "featured_link": null, + "has_accepted_answer": false, + "posters": [ + { + "extras": "latest", + "description": "Original Poster, Most Recent Poster", + "user_id": 3, + "primary_group_id": null + }, + { + "extras": null, + "description": "Frequent Poster", + "user_id": 2, + "primary_group_id": null + } + ] + }, + { + "id": 11, + "title": "My First Test Post", + "fancy_title": "My First Test Post", + "slug": "my-first-test-post", + "posts_count": 3, + "reply_count": 0, + "highest_post_number": 3, + "image_url": null, + "created_at": "2019-08-02T11:12:29.408Z", + "last_posted_at": "2019-08-04T20:50:39.667Z", + "bumped": true, + "bumped_at": "2019-08-04T20:50:39.667Z", + "unseen": false, + "pinned": false, + "unpinned": null, + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "views": 8, + "like_count": 1, + "has_summary": false, + "archetype": "regular", + "last_poster_username": "d11", + "category_id": 1, + "pinned_globally": false, + "featured_link": null, + "has_accepted_answer": false, + "posters": [ + { + "extras": "latest", + "description": "Original Poster, Most Recent Poster", + "user_id": 2, + "primary_group_id": null + }, + { + "extras": null, + "description": "Frequent Poster", + "user_id": 3, + "primary_group_id": null + } + ] + }, + { + "id": 7, + "title": "Welcome to Discourse", + "fancy_title": "Welcome to Discourse", + "slug": "welcome-to-discourse", + "posts_count": 1, + "reply_count": 0, + "highest_post_number": 1, + "image_url": "https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif", + "created_at": "2019-04-02T20:20:12.879Z", + "last_posted_at": "2019-04-02T20:20:12.933Z", + "bumped": true, + "bumped_at": "2019-04-10T23:16:19.399Z", + "unseen": false, + "pinned": true, + "unpinned": null, + "excerpt": "This is a test instance. \nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It’s important! \nEdit this into a brief description of your community: \n\nWho i…", + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "views": 1, + "like_count": 0, + "has_summary": false, + "archetype": "regular", + "last_poster_username": "system", + "category_id": 1, + "pinned_globally": true, + "featured_link": null, + "has_accepted_answer": false, + "posters": [ + { + "extras": "latest single", + "description": "Original Poster, Most Recent Poster", + "user_id": -1, + "primary_group_id": null + } + ] + } + ] + } +} diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLmpzb24 b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLmpzb24 new file mode 100644 index 0000000..769157f --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLmpzb24 @@ -0,0 +1,536 @@ +{ + "latest_posts": [ + { + "id": 22, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-04T20:50:39.667Z", + "cooked": "

Adding another post, after having deleted a whole thread.

", + "post_number": 3, + "post_type": 1, + "updated_at": "2019-08-04T20:50:39.667Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 1, + "score": 0.2, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "topic_title": "My First Test Post", + "topic_html_title": "My First Test Post", + "category_id": 1, + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "Adding another post, after having deleted a whole thread.", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 18, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-02T11:16:31.313Z", + "cooked": "\n

Excellent link. I’ve quoted you.

", + "post_number": 2, + "post_type": 1, + "updated_at": "2019-08-02T11:17:12.745Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 1, + "incoming_link_count": 0, + "reads": 1, + "score": 0.2, + "yours": false, + "topic_id": 13, + "topic_slug": "a-thread-with-links", + "topic_title": "A Thread With Links", + "topic_html_title": "A Thread With Links", + "category_id": 1, + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 3, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "[quote=\"dl-proto, post:1, topic:13\"]\nHere is a link to another discourse thread:\n[/quote]\n\nExcellent link. I've quoted you.", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 17, + "name": "dandelion protocol acct", + "username": "dl-proto", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png", + "created_at": "2019-08-02T11:15:53.905Z", + "cooked": "

Here is a link to another discourse thread: My First Test Post

\n

Here is a link to a GitHub issue: https://github.com/sourcecred-test/example-github/issues/1

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-08-02T11:15:53.905Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 0.4, + "yours": false, + "topic_id": 13, + "topic_slug": "a-thread-with-links", + "topic_title": "A Thread With Links", + "topic_html_title": "A Thread With Links", + "category_id": 1, + "display_username": "dandelion protocol acct", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "Here is a link to another discourse thread: https://sourcecred-test.discourse.group/t/my-first-test-post/11/3\n\nHere is a link to a GitHub issue: https://github.com/sourcecred-test/example-github/issues/1", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": false, + "staff": false, + "user_id": 3, + "hidden": false, + "trust_level": 1, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-08-02T11:14:13.800Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 16, + "name": "dandelion protocol acct", + "username": "dl-proto", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png", + "created_at": "2019-08-02T11:14:56.285Z", + "cooked": "

Ah, an excellent post. I will like it and reply.

", + "post_number": 2, + "post_type": 1, + "updated_at": "2019-08-02T11:14:56.285Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 0.4, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "topic_title": "My First Test Post", + "topic_html_title": "My First Test Post", + "category_id": 1, + "display_username": "dandelion protocol acct", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "Ah, an excellent post. I will like it and reply.", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": false, + "staff": false, + "user_id": 3, + "hidden": false, + "trust_level": 1, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-08-02T11:14:13.800Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 14, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-02T11:12:29.476Z", + "cooked": "

This is a test post.

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-08-02T11:12:29.476Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 15.4, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "topic_title": "My First Test Post", + "topic_html_title": "My First Test Post", + "category_id": 1, + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "This is a test post.", + "actions_summary": [ + { + "id": 2, + "count": 1, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 10, + "name": "system", + "username": "system", + "avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png", + "created_at": "2019-04-02T20:20:12.933Z", + "cooked": "

This is a test instance.
\nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It’s important!

\n

Edit this into a brief description of your community:

\n
    \n
  • Who is it for?
  • \n
  • What can they find here?
  • \n
  • Why should they come here?
  • \n
  • Where can they read more (links, resources, etc)?
  • \n
\n

\n

You may want to close this topic via the admin \":wrench:\" (at the upper right and bottom), so that replies don’t pile up on an announcement.

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-04-10T23:16:19.366Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 1, + "score": 0.2, + "yours": false, + "topic_id": 7, + "topic_slug": "welcome-to-discourse", + "topic_title": "Welcome to Discourse", + "topic_html_title": "Welcome to Discourse", + "category_id": 1, + "display_username": "system", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 2, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "This is a test instance.\nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It's important!\n\n**Edit this** into a brief description of your community:\n\n- Who is it for?\n- What can they find here?\n- Why should they come here?\n- Where can they read more (links, resources, etc)?\n\n\n\nYou may want to close this topic via the admin :wrench: (at the upper right and bottom), so that replies don't pile up on an announcement.", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": true, + "admin": true, + "staff": true, + "user_id": -1, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:20:06.287Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 1, + "name": "system", + "username": "system", + "avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png", + "created_at": "2019-04-02T20:20:09.844Z", + "cooked": "

Discussion about this site, its organization, how it works, and how we can improve it.

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-04-02T20:20:09.844Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 1, + "score": 0.2, + "yours": false, + "topic_id": 1, + "topic_slug": "about-the-site-feedback-category", + "topic_title": "About the Site Feedback category", + "topic_html_title": "About the Site Feedback category", + "category_id": 2, + "display_username": "system", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "Discussion about this site, its organization, how it works, and how we can improve it.", + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": true, + "admin": true, + "staff": true, + "user_id": -1, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:20:06.287Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + } + ] +} diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLzE0Lmpzb24 b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLzE0Lmpzb24 new file mode 100644 index 0000000..a525c63 --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3Bvc3RzLzE0Lmpzb24 @@ -0,0 +1,75 @@ +{ + "id": 14, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-02T11:12:29.476Z", + "cooked": "

This is a test post.

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-08-02T11:12:29.476Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 15.4, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "user_title": null, + "raw": "This is a test post.", + "actions_summary": [ + { + "id": 2, + "count": 1, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false +} diff --git a/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3QvMTEuanNvbg b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3QvMTEuanNvbg new file mode 100644 index 0000000..752a42d --- /dev/null +++ b/src/plugins/discourse/snapshots/aHR0cHM6Ly9zb3VyY2VjcmVkLXRlc3QuZGlzY291cnNlLmdyb3VwL3QvMTEuanNvbg @@ -0,0 +1,437 @@ +{ + "post_stream": { + "posts": [ + { + "id": 14, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-02T11:12:29.476Z", + "cooked": "

This is a test post.

", + "post_number": 1, + "post_type": 1, + "updated_at": "2019-08-02T11:12:29.476Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 15.4, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "read": false, + "user_title": null, + "actions_summary": [ + { + "id": 2, + "count": 1, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 16, + "name": "dandelion protocol acct", + "username": "dl-proto", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png", + "created_at": "2019-08-02T11:14:56.285Z", + "cooked": "

Ah, an excellent post. I will like it and reply.

", + "post_number": 2, + "post_type": 1, + "updated_at": "2019-08-02T11:14:56.285Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 2, + "score": 0.4, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "display_username": "dandelion protocol acct", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "read": false, + "user_title": null, + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": false, + "staff": false, + "user_id": 3, + "hidden": false, + "trust_level": 1, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-08-02T11:14:13.800Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + }, + { + "id": 22, + "name": "D", + "username": "d11", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "created_at": "2019-08-04T20:50:39.667Z", + "cooked": "

Adding another post, after having deleted a whole thread.

", + "post_number": 3, + "post_type": 1, + "updated_at": "2019-08-04T20:50:39.667Z", + "reply_count": 0, + "reply_to_post_number": null, + "quote_count": 0, + "incoming_link_count": 0, + "reads": 1, + "score": 0.2, + "yours": false, + "topic_id": 11, + "topic_slug": "my-first-test-post", + "display_username": "D", + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_bg_color": null, + "primary_group_flair_color": null, + "version": 1, + "can_edit": false, + "can_delete": false, + "can_recover": false, + "can_wiki": false, + "read": false, + "user_title": null, + "actions_summary": [ + { + "id": 2, + "can_act": true + }, + { + "id": 3, + "can_act": true + }, + { + "id": 4, + "can_act": true + }, + { + "id": 8, + "can_act": true + }, + { + "id": 6, + "can_act": true + }, + { + "id": 7, + "can_act": true + } + ], + "moderator": false, + "admin": true, + "staff": true, + "user_id": 2, + "hidden": false, + "trust_level": 4, + "deleted_at": null, + "user_deleted": false, + "edit_reason": null, + "can_view_edit_history": true, + "wiki": false, + "user_created_at": "2019-04-02T20:22:14.312Z", + "user_date_of_birth": null, + "can_accept_answer": false, + "can_unaccept_answer": false, + "accepted_answer": false + } + ], + "stream": [ + 14, + 16, + 22 + ] + }, + "timeline_lookup": [ + [ + 1, + 12 + ], + [ + 3, + 9 + ] + ], + "suggested_topics": [ + { + "id": 13, + "title": "A Thread With Links", + "fancy_title": "A Thread With Links", + "slug": "a-thread-with-links", + "posts_count": 2, + "reply_count": 0, + "highest_post_number": 2, + "image_url": null, + "created_at": "2019-08-02T11:15:53.843Z", + "last_posted_at": "2019-08-02T11:16:31.313Z", + "bumped": true, + "bumped_at": "2019-08-02T11:17:12.755Z", + "unseen": false, + "pinned": false, + "unpinned": null, + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "archetype": "regular", + "like_count": 0, + "views": 6, + "category_id": 1, + "featured_link": null, + "has_accepted_answer": false, + "posters": [ + { + "extras": "latest", + "description": "Original Poster, Most Recent Poster", + "user": { + "id": 3, + "username": "dl-proto", + "name": "dandelion protocol acct", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png" + } + }, + { + "extras": null, + "description": "Frequent Poster", + "user": { + "id": 2, + "username": "d11", + "name": "D", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png" + } + } + ] + }, + { + "id": 7, + "title": "Welcome to Discourse", + "fancy_title": "Welcome to Discourse", + "slug": "welcome-to-discourse", + "posts_count": 1, + "reply_count": 0, + "highest_post_number": 1, + "image_url": "https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif", + "created_at": "2019-04-02T20:20:12.879Z", + "last_posted_at": "2019-04-02T20:20:12.933Z", + "bumped": true, + "bumped_at": "2019-04-10T23:16:19.399Z", + "unseen": false, + "pinned": true, + "unpinned": null, + "excerpt": "This is a test instance. \nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It’s important! \nEdit this into a brief description of your community: \n\nWho i…", + "visible": true, + "closed": false, + "archived": false, + "bookmarked": null, + "liked": null, + "archetype": "regular", + "like_count": 0, + "views": 1, + "category_id": 1, + "featured_link": null, + "has_accepted_answer": false, + "posters": [ + { + "extras": "latest single", + "description": "Original Poster, Most Recent Poster", + "user": { + "id": -1, + "username": "system", + "name": "system", + "avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png" + } + } + ] + } + ], + "id": 11, + "title": "My First Test Post", + "fancy_title": "My First Test Post", + "posts_count": 3, + "created_at": "2019-08-02T11:12:29.408Z", + "views": 8, + "reply_count": 0, + "like_count": 1, + "last_posted_at": "2019-08-04T20:50:39.667Z", + "visible": true, + "closed": false, + "archived": false, + "has_summary": false, + "archetype": "regular", + "slug": "my-first-test-post", + "category_id": 1, + "word_count": 24, + "deleted_at": null, + "user_id": 2, + "featured_link": null, + "pinned_globally": false, + "pinned_at": null, + "pinned_until": null, + "draft": null, + "draft_key": "topic_11", + "draft_sequence": 0, + "unpinned": null, + "pinned": false, + "current_post_number": 1, + "highest_post_number": 3, + "deleted_by": null, + "actions_summary": [ + { + "id": 4, + "count": 0, + "hidden": false, + "can_act": true + }, + { + "id": 8, + "count": 0, + "hidden": false, + "can_act": true + }, + { + "id": 7, + "count": 0, + "hidden": false, + "can_act": true + } + ], + "chunk_size": 20, + "bookmarked": null, + "topic_timer": null, + "private_topic_timer": null, + "message_bus_last_id": 0, + "participant_count": 2, + "tags_disable_ads": false, + "details": { + "notification_level": 1, + "can_create_post": true, + "can_reply_as_new_topic": true, + "can_flag_topic": true, + "participants": [ + { + "id": 2, + "username": "d11", + "name": "D", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png", + "post_count": 2, + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_color": null, + "primary_group_flair_bg_color": null + }, + { + "id": 3, + "username": "dl-proto", + "name": "dandelion protocol acct", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png", + "post_count": 1, + "primary_group_name": null, + "primary_group_flair_url": null, + "primary_group_flair_color": null, + "primary_group_flair_bg_color": null + } + ], + "created_by": { + "id": 2, + "username": "d11", + "name": "D", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png" + }, + "last_poster": { + "id": 2, + "username": "d11", + "name": "D", + "avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png" + } + } +} diff --git a/src/plugins/discourse/update_discourse_api_snapshots.sh b/src/plugins/discourse/update_discourse_api_snapshots.sh new file mode 100755 index 0000000..b44baa2 --- /dev/null +++ b/src/plugins/discourse/update_discourse_api_snapshots.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -eu + +snapshots_dir=src/plugins/discourse/snapshots +test_instance_url="https://sourcecred-test.discourse.group" +test_instance_username="credbot" + +if [ -z "${DISCOURSE_TEST_API_KEY:-}" ]; then + printf >&2 'Please set the DISCOURSE_TEST_API_KEY environment variable.\n' + printf >&2 'Contact the SourceCred maintainers to get the key.\n' + exit 1 +fi + +if [ ! "$(jq --version)" ]; then + printf >&2 'This script depends on jq. Please install it.\n' + exit 1 +fi + +toplevel="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" +cd "${toplevel}" + +fetch() { + url="${test_instance_url}$1" + filename="$(printf '%s' "${url}" | base64 -w 0 | tr -d '=' | tr '/+' '_-')" + path="${snapshots_dir}/${filename}" + curl -sfL "$url" \ + -H "Api-Key: ${DISCOURSE_TEST_API_KEY}" \ + -H "Api-Username: ${test_instance_username}" \ + -H "Accept: application/json" \ + | jq '.' > "${path}" +} + +rm -r "${snapshots_dir}" +mkdir "${snapshots_dir}" +fetch "/latest.json?order=created" +fetch "/posts.json" +fetch "/t/11.json" +fetch "/posts/14.json"