Add class for fetching data from Discourse (#1265)

The `DiscourseFetcher` class abstracts over fetching from the Discourse
API, and post-processing and filtering the result into a form that's
convenient for us.

Testing is a bit tricky because the Discourse API keys are sensitive
(they are admin keys) and so I'm reluctant to commit them, even for our
test instance. As a workaround, I've added a shell script which
downloads some data from the SourceCred test instance, and saves it with
a filename which is an encoding of the actual endpoint. Then, in
testing, we can use a mocked fetch which actually hits the snapshots
directory, and thus validate the processing logic on "real" data from
the server. We also test that the fetch headers are set correctly, and
that we handle non-200 error codes appropriately.

Test plan: In addition to the included tests, I have an end-to-end test
which actually uses this fetcher to fully populate the mirror and then
generate a valid SourceCred graph.

This builds on API investigations
[here](https://github.com/sourcecred/sourcecred/issues/865#issuecomment-478026449),
and is general progress towards #865. Thanks to @erlend-sh, without whom
we wouldn't have a test instance.
This commit is contained in:
Dandelion Mané 2019-08-15 13:22:06 +02:00 committed by GitHub
parent 610ace0cff
commit fd95be68a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1688 additions and 0 deletions

View File

@ -34,5 +34,12 @@ echo "Updating github/fetchGithubOrgTest.sh"
echo "Updating github/fetchGithubRepoTest.sh"
./src/plugins/github/fetchGithubRepoTest.sh -u --no-build
if [ -z "${DISCOURSE_TEST_API_KEY:-}" ]; then
echo "Updating Discourse API snapshots"
./src/plugins/discourse/update_discourse_api_snapshots.sh
else
echo "Not updating Discourse API snapshots (need DISCOURSE_TEST_API_KEY)"
fi
echo "Updating Jest snapshots"
yarn unit -u

View File

@ -0,0 +1,110 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`plugins/discourse/fetch snapshot testing loads a particular post from snapshot 1`] = `
Object {
"authorUsername": "d11",
"id": 14,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1564744349476,
"topicId": 11,
}
`;
exports[`plugins/discourse/fetch snapshot testing loads a particular topic from snapshot 1`] = `
Object {
"posts": Array [
Object {
"authorUsername": "d11",
"id": 14,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1564744349476,
"topicId": 11,
},
Object {
"authorUsername": "dl-proto",
"id": 16,
"indexWithinTopic": 2,
"replyToPostIndex": null,
"timestampMs": 1564744496285,
"topicId": 11,
},
Object {
"authorUsername": "d11",
"id": 22,
"indexWithinTopic": 3,
"replyToPostIndex": null,
"timestampMs": 1564951839667,
"topicId": 11,
},
],
"topic": Object {
"authorUsername": "d11",
"id": 11,
"timestampMs": 1564744349408,
"title": "My First Test Post",
},
}
`;
exports[`plugins/discourse/fetch snapshot testing loads latest posts from snapshot 1`] = `
Array [
Object {
"authorUsername": "d11",
"id": 22,
"indexWithinTopic": 3,
"replyToPostIndex": null,
"timestampMs": 1564951839667,
"topicId": 11,
},
Object {
"authorUsername": "d11",
"id": 18,
"indexWithinTopic": 2,
"replyToPostIndex": null,
"timestampMs": 1564744591313,
"topicId": 13,
},
Object {
"authorUsername": "dl-proto",
"id": 17,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1564744553905,
"topicId": 13,
},
Object {
"authorUsername": "dl-proto",
"id": 16,
"indexWithinTopic": 2,
"replyToPostIndex": null,
"timestampMs": 1564744496285,
"topicId": 11,
},
Object {
"authorUsername": "d11",
"id": 14,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1564744349476,
"topicId": 11,
},
Object {
"authorUsername": "system",
"id": 10,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1554236412933,
"topicId": 7,
},
Object {
"authorUsername": "system",
"id": 1,
"indexWithinTopic": 1,
"replyToPostIndex": null,
"timestampMs": 1554236409844,
"topicId": 1,
},
]
`;

View File

@ -0,0 +1,218 @@
// @flow
/**
* Class for retrieving data from the Discourse API.
*
* The Discourse API implements the JSON endpoints for all functionality of the actual site.
* As such, it tends to return a lot of information that we don't care about (in contrast
* to a GraphQL API which would give us only what we ask for). As such, we implement a simple
* interface over it, which both abstracts over calling the API, and does some post-processing
* on the results to simplify it to data that is relevant for us.
*/
import stringify from "json-stable-stringify";
import fetch from "isomorphic-fetch";
export type UserId = number;
export type PostId = number;
export type TopicId = number;
export type Topic = {|
+id: TopicId,
+title: string,
+timestampMs: number,
+authorUsername: string,
|};
export type Post = {|
+id: PostId,
+topicId: TopicId,
// Which number post this was within the topic (starts at 1)
+indexWithinTopic: number,
// The indexWithinTopic of the post within the same topic that this post was a
// reply to. Will be `null` if this post was the first post, or if it was a
// reply to the first post.
+replyToPostIndex: number | null,
+timestampMs: number,
+authorUsername: string,
|};
export type TopicWithPosts = {|
+topic: Topic,
// Not guaranteed to contain all the Posts in the topic—clients will need to
// manually request some posts. The raw response actually includes a list of
// all the PostIds in the topic, but for now we don't use them.
//
// We do use these Posts though, as it allows us to save requesting them all
// individually.
+posts: $ReadOnlyArray<Post>,
|};
/**
* Interface over the external Discourse API, structured to suit our particular needs.
* We have an interface (as opposed to just an implementation) to enable easy mocking and
* testing.
*/
export interface Discourse {
// Get the `id` of the latest topic on the server.
// Vital so that we can then enumerate and fetch every Topic we haven't seen yet.
// May reject on not OK status like 404 or 403.
latestTopicId(): Promise<TopicId>;
// Retrieve the Topic with Posts for a given id.
// Will resolve to null if the response status is 403 or 404. 403 because the
// topic may be hidden from the API user; 404 because we sometimes see
// 404s in prod and want to ignore those topic ids. (Not sure why it happens.)
// May reject if the status is not OK and is not 404 or 403.
topicWithPosts(id: TopicId): Promise<TopicWithPosts | null>;
// Retrieve an individual Post by its id.
// Will resolve to null if the response status is 403 or 404. 403 because the
// topic may be hidden from the API user; 404 because we sometimes see
// 404s in prod and want to ignore those topic ids. (Not sure why it happens.)
// May reject if the status is not OK and is not 404 or 403.
post(id: PostId): Promise<Post | null>;
// Retrieve the latest posts from the server.
// Vital so that we can then enumerate and fetch every Post that we haven't
// encountered.
// May reject on not OK status like 404 or 403.
latestPosts(): Promise<Post[]>;
}
export class Fetcher implements Discourse {
+options: DiscourseFetchOptions;
+_fetchImplementation: typeof fetch;
constructor(
options: DiscourseFetchOptions,
// fetchImplementation shouldn't be provided by clients, but is convenient for testing.
fetchImplementation?: typeof fetch
) {
this.options = options;
this._fetchImplementation = fetchImplementation || fetch;
}
_fetch(endpoint: string): Promise<Response> {
const {serverUrl, apiKey, apiUsername} = this.options;
if (!endpoint.startsWith("/")) {
throw new Error(`invalid endpoint: ${endpoint}`);
}
if (!serverUrl.startsWith("http") || serverUrl.endsWith("/")) {
throw new Error(`invalid server url: ${serverUrl}`);
}
const fetchOptions = {
method: "GET",
headers: {
"Api-Key": apiKey,
"Api-Username": apiUsername,
Accept: "application/json",
},
};
const fullUrl = `${serverUrl}${endpoint}`;
return this._fetchImplementation(fullUrl, fetchOptions);
}
async latestTopicId(): Promise<TopicId> {
const response = await this._fetch("/latest.json?order=created");
maybeFail404(response);
maybeFail403(response);
if (!response.ok) {
throw new Error(`not OK status ${response.status} on ${response.url}`);
}
const json = await response.json();
if (json.topic_list.topics.length === 0) {
throw new Error(`no topics! got ${stringify(json)} as latest topics.`);
}
return json.topic_list.topics[0].id;
}
async latestPosts(): Promise<Post[]> {
const response = await this._fetch("/posts.json");
maybeFail404(response);
maybeFail403(response);
if (!response.ok) {
throw new Error(`not OK status ${response.status} on ${response.url}`);
}
const json = await response.json();
return json.latest_posts.map(parsePost);
}
async topicWithPosts(id: TopicId): Promise<TopicWithPosts | null> {
const response = await this._fetch(`/t/${id}.json`);
if (response.status === 404) {
// Not sure why this happens, but a topic can sometimes 404.
// We should just consider it unreachable.
// Here is an example: https://discourse.sourcecred.io/t/116
return null;
}
if (response.status === 403) {
// Probably this topic is hidden or deleted.
// Just consider it unreachable.
// If the issue is that the user provided bad keys, then
// they will get a more helpful error when they try to get the latest
// topic id.
return null;
}
if (response.status !== 200) {
throw new Error(`not OK status ${response.status} on ${response.url}`);
}
const json = await response.json();
const posts = json.post_stream.posts.map(parsePost);
const topic: Topic = {
id: json.id,
title: json.title,
timestampMs: +new Date(json.created_at),
authorUsername: json.details.created_by.username,
};
return {topic, posts};
}
async post(id: PostId): Promise<Post | null> {
const response = await this._fetch(`/posts/${id}.json`);
if (response.status === 404) {
// Since topics can 404, I assume posts can too.
return null;
}
if (response.status === 403) {
// Probably this post is hidden or deleted.
return null;
}
if (response.status !== 200) {
throw new Error(`not OK status ${response.status} on ${response.url}`);
}
const json = await response.json();
return parsePost(json);
}
}
function maybeFail404(response) {
if (response.status === 404) {
throw new Error(`404 Not Found on: ${response.url}; maybe bad serverUrl?`);
}
}
function maybeFail403(response) {
if (response.status === 403) {
throw new Error(`403 Forbidden: bad API username or key?`);
}
}
function parsePost(json: any): Post {
return {
id: json.id,
timestampMs: Date.parse(json.created_at),
indexWithinTopic: json.post_number,
replyToPostIndex: json.reply_to_post_number,
topicId: json.topic_id,
authorUsername: json.username,
};
}
export type DiscourseFetchOptions = {|
apiKey: string,
// We'll use the view permissions for this user. It needs to be a real user
// on the server. I recommend making a new user called "credbot" with no
// special permissions for this purpose. If you use a permissioned user (e.g.
// "system") then SourceCred will pick up hidden and deleted posts,
// potentially leaking private information.
apiUsername: string,
serverUrl: string,
|};

View File

@ -0,0 +1,104 @@
// @flow
import deepFreeze from "deep-freeze";
import {Fetcher, type DiscourseFetchOptions} from "./fetch";
import base64url from "base64url";
import path from "path";
import fs from "fs-extra";
describe("plugins/discourse/fetch", () => {
const options: DiscourseFetchOptions = deepFreeze({
apiKey: "FAKE_KEY",
apiUsername: "credbot",
serverUrl: "https://sourcecred-test.discourse.group",
});
describe("snapshot testing", () => {
async function snapshotFetch(
url: string | Request | URL
): Promise<Response> {
const snapshotDir = "src/plugins/discourse/snapshots";
const filename = base64url(url);
const file = path.join(snapshotDir, filename);
if (await fs.exists(file)) {
const contents = await fs.readFile(file);
return new Response(contents, {status: 200});
} else {
throw new Error(`couldn't load snapshot for ${file}`);
}
}
const snapshotFetcher = () => new Fetcher(options, snapshotFetch);
it("loads LatestTopicId from snapshot", async () => {
const topicId = await snapshotFetcher().latestTopicId();
expect(topicId).toMatchInlineSnapshot(`13`);
});
it("loads latest posts from snapshot", async () => {
expect(await snapshotFetcher().latestPosts()).toMatchSnapshot();
});
it("loads a particular topic from snapshot", async () => {
expect(await snapshotFetcher().topicWithPosts(11)).toMatchSnapshot();
});
it("loads a particular post from snapshot", async () => {
expect(await snapshotFetcher().post(14)).toMatchSnapshot();
});
});
describe("error handling", () => {
const fakeFetch = (status: number) => (url: any) => {
const resp = new Response("", {status, url});
return Promise.resolve(resp);
};
const fetcherWithStatus = (status: number) =>
new Fetcher(options, fakeFetch(status));
function expectError(name, f, status) {
it(`${name} errors on ${String(status)}`, () => {
const fetcher = fetcherWithStatus(status);
expect.assertions(1);
const result = f(fetcher);
return result.catch((e) => expect(e.message).toMatch(String(status)));
});
}
expectError("latestTopicId", (x) => x.latestTopicId(), 404);
expectError("latestTopicId", (x) => x.latestTopicId(), 403);
expectError("latestTopicId", (x) => x.latestTopicId(), 429);
expectError("latestPosts", (x) => x.latestPosts(), 404);
expectError("latestPosts", (x) => x.latestPosts(), 403);
expectError("latestPosts", (x) => x.latestPosts(), 429);
expectError("topic", (x) => x.topicWithPosts(14), 429);
expectError("post", (x) => x.post(14), 429);
function expectNull(name, f, status) {
it(`${name} returns null on ${String(status)}`, async () => {
const fetcher = fetcherWithStatus(status);
const result = f(fetcher);
expect(await result).toBe(null);
});
}
expectNull("topic", (x) => x.topicWithPosts(14), 404);
expectNull("topic", (x) => x.topicWithPosts(14), 403);
expectNull("post", (x) => x.post(14), 404);
expectNull("post", (x) => x.post(14), 403);
});
describe("fetch headers", () => {
it("calls fetch with the right options and headers", async () => {
let fetchOptions: ?RequestOptions;
const fakeFetch = (url, _options) => {
fetchOptions = _options;
return Promise.resolve(new Response("", {status: 404}));
};
await new Fetcher(options, fakeFetch).post(1337);
if (fetchOptions == null) {
throw new Error("fetchOptions == null");
}
expect(fetchOptions.method).toEqual("GET");
expect(fetchOptions.headers["Api-Key"]).toEqual(options.apiKey);
expect(fetchOptions.headers["Api-Username"]).toEqual(options.apiUsername);
expect(fetchOptions.headers["Accept"]).toEqual("application/json");
});
});
});

View File

@ -0,0 +1,162 @@
{
"users": [
{
"id": 3,
"username": "dl-proto",
"name": "dandelion protocol acct",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png"
},
{
"id": 2,
"username": "d11",
"name": "D",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png"
},
{
"id": -1,
"username": "system",
"name": "system",
"avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png"
}
],
"primary_groups": [],
"topic_list": {
"can_create_topic": true,
"draft": null,
"draft_key": "new_topic",
"draft_sequence": 0,
"per_page": 30,
"topics": [
{
"id": 13,
"title": "A Thread With Links",
"fancy_title": "A Thread With Links",
"slug": "a-thread-with-links",
"posts_count": 2,
"reply_count": 0,
"highest_post_number": 2,
"image_url": null,
"created_at": "2019-08-02T11:15:53.843Z",
"last_posted_at": "2019-08-02T11:16:31.313Z",
"bumped": true,
"bumped_at": "2019-08-02T11:17:12.755Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"views": 6,
"like_count": 0,
"has_summary": false,
"archetype": "regular",
"last_poster_username": "dl-proto",
"category_id": 1,
"pinned_globally": false,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest",
"description": "Original Poster, Most Recent Poster",
"user_id": 3,
"primary_group_id": null
},
{
"extras": null,
"description": "Frequent Poster",
"user_id": 2,
"primary_group_id": null
}
]
},
{
"id": 11,
"title": "My First Test Post",
"fancy_title": "My First Test Post",
"slug": "my-first-test-post",
"posts_count": 3,
"reply_count": 0,
"highest_post_number": 3,
"image_url": null,
"created_at": "2019-08-02T11:12:29.408Z",
"last_posted_at": "2019-08-04T20:50:39.667Z",
"bumped": true,
"bumped_at": "2019-08-04T20:50:39.667Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"views": 8,
"like_count": 1,
"has_summary": false,
"archetype": "regular",
"last_poster_username": "d11",
"category_id": 1,
"pinned_globally": false,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest",
"description": "Original Poster, Most Recent Poster",
"user_id": 2,
"primary_group_id": null
},
{
"extras": null,
"description": "Frequent Poster",
"user_id": 3,
"primary_group_id": null
}
]
},
{
"id": 7,
"title": "Welcome to Discourse",
"fancy_title": "Welcome to Discourse",
"slug": "welcome-to-discourse",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": "https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif",
"created_at": "2019-04-02T20:20:12.879Z",
"last_posted_at": "2019-04-02T20:20:12.933Z",
"bumped": true,
"bumped_at": "2019-04-10T23:16:19.399Z",
"unseen": false,
"pinned": true,
"unpinned": null,
"excerpt": "This is a test instance. \nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. Its important! \nEdit this into a brief description of your community: \n\nWho i&hellip;",
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"views": 1,
"like_count": 0,
"has_summary": false,
"archetype": "regular",
"last_poster_username": "system",
"category_id": 1,
"pinned_globally": true,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user_id": -1,
"primary_group_id": null
}
]
}
]
}
}

View File

@ -0,0 +1,536 @@
{
"latest_posts": [
{
"id": 22,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-04T20:50:39.667Z",
"cooked": "<p>Adding another post, after having deleted a whole thread.</p>",
"post_number": 3,
"post_type": 1,
"updated_at": "2019-08-04T20:50:39.667Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 1,
"score": 0.2,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"topic_title": "My First Test Post",
"topic_html_title": "My First Test Post",
"category_id": 1,
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "Adding another post, after having deleted a whole thread.",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 18,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-02T11:16:31.313Z",
"cooked": "<aside class=\"quote no-group\" data-post=\"1\" data-topic=\"13\">\n<div class=\"title\">\n<div class=\"quote-controls\"></div>\n<img alt width=\"20\" height=\"20\" src=\"https://avatars.discourse.org/v4/letter/d/8797f3/40.png\" class=\"avatar\"> dl-proto:</div>\n<blockquote>\n<p>Here is a link to another discourse thread:</p>\n</blockquote>\n</aside>\n<p>Excellent link. Ive quoted you.</p>",
"post_number": 2,
"post_type": 1,
"updated_at": "2019-08-02T11:17:12.745Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 1,
"incoming_link_count": 0,
"reads": 1,
"score": 0.2,
"yours": false,
"topic_id": 13,
"topic_slug": "a-thread-with-links",
"topic_title": "A Thread With Links",
"topic_html_title": "A Thread With Links",
"category_id": 1,
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 3,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "[quote=\"dl-proto, post:1, topic:13\"]\nHere is a link to another discourse thread:\n[/quote]\n\nExcellent link. I've quoted you.",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 17,
"name": "dandelion protocol acct",
"username": "dl-proto",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png",
"created_at": "2019-08-02T11:15:53.905Z",
"cooked": "<p>Here is a link to another discourse thread: <a href=\"https://sourcecred-test.discourse.group/t/my-first-test-post/11/3\" class=\"inline-onebox\">My First Test Post</a></p>\n<p>Here is a link to a GitHub issue: <a href=\"https://github.com/sourcecred-test/example-github/issues/1\" rel=\"nofollow noopener\">https://github.com/sourcecred-test/example-github/issues/1</a></p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-08-02T11:15:53.905Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 0.4,
"yours": false,
"topic_id": 13,
"topic_slug": "a-thread-with-links",
"topic_title": "A Thread With Links",
"topic_html_title": "A Thread With Links",
"category_id": 1,
"display_username": "dandelion protocol acct",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "Here is a link to another discourse thread: https://sourcecred-test.discourse.group/t/my-first-test-post/11/3\n\nHere is a link to a GitHub issue: https://github.com/sourcecred-test/example-github/issues/1",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": false,
"staff": false,
"user_id": 3,
"hidden": false,
"trust_level": 1,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-08-02T11:14:13.800Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 16,
"name": "dandelion protocol acct",
"username": "dl-proto",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png",
"created_at": "2019-08-02T11:14:56.285Z",
"cooked": "<p>Ah, an excellent post. I will like it and reply.</p>",
"post_number": 2,
"post_type": 1,
"updated_at": "2019-08-02T11:14:56.285Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 0.4,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"topic_title": "My First Test Post",
"topic_html_title": "My First Test Post",
"category_id": 1,
"display_username": "dandelion protocol acct",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "Ah, an excellent post. I will like it and reply.",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": false,
"staff": false,
"user_id": 3,
"hidden": false,
"trust_level": 1,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-08-02T11:14:13.800Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 14,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-02T11:12:29.476Z",
"cooked": "<p>This is a test post.</p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-08-02T11:12:29.476Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 15.4,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"topic_title": "My First Test Post",
"topic_html_title": "My First Test Post",
"category_id": 1,
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "This is a test post.",
"actions_summary": [
{
"id": 2,
"count": 1,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 10,
"name": "system",
"username": "system",
"avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png",
"created_at": "2019-04-02T20:20:12.933Z",
"cooked": "<p>This is a test instance.<br>\nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. Its important!</p>\n<p><strong>Edit this</strong> into a brief description of your community:</p>\n<ul>\n<li>Who is it for?</li>\n<li>What can they find here?</li>\n<li>Why should they come here?</li>\n<li>Where can they read more (links, resources, etc)?</li>\n</ul>\n<p><img src=\"https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif\" width=\"508\" height=\"106\"></p>\n<p>You may want to close this topic via the admin <img src=\"https://sjc3.discourse-cdn.com/free1/images/emoji/twitter/wrench.png?v=9\" title=\":wrench:\" class=\"emoji\" alt=\":wrench:\"> (at the upper right and bottom), so that replies dont pile up on an announcement.</p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-04-10T23:16:19.366Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 1,
"score": 0.2,
"yours": false,
"topic_id": 7,
"topic_slug": "welcome-to-discourse",
"topic_title": "Welcome to Discourse",
"topic_html_title": "Welcome to Discourse",
"category_id": 1,
"display_username": "system",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 2,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "This is a test instance.\nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. It's important!\n\n**Edit this** into a brief description of your community:\n\n- Who is it for?\n- What can they find here?\n- Why should they come here?\n- Where can they read more (links, resources, etc)?\n\n<img src=\"/images/welcome/discourse-edit-post-animated.gif\" width=\"508\" height=\"106\">\n\nYou may want to close this topic via the admin :wrench: (at the upper right and bottom), so that replies don't pile up on an announcement.",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": true,
"admin": true,
"staff": true,
"user_id": -1,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:20:06.287Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 1,
"name": "system",
"username": "system",
"avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png",
"created_at": "2019-04-02T20:20:09.844Z",
"cooked": "<p>Discussion about this site, its organization, how it works, and how we can improve it.</p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-04-02T20:20:09.844Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 1,
"score": 0.2,
"yours": false,
"topic_id": 1,
"topic_slug": "about-the-site-feedback-category",
"topic_title": "About the Site Feedback category",
"topic_html_title": "About the Site Feedback category",
"category_id": 2,
"display_username": "system",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "Discussion about this site, its organization, how it works, and how we can improve it.",
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": true,
"admin": true,
"staff": true,
"user_id": -1,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:20:06.287Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
}
]
}

View File

@ -0,0 +1,75 @@
{
"id": 14,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-02T11:12:29.476Z",
"cooked": "<p>This is a test post.</p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-08-02T11:12:29.476Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 15.4,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"user_title": null,
"raw": "This is a test post.",
"actions_summary": [
{
"id": 2,
"count": 1,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
}

View File

@ -0,0 +1,437 @@
{
"post_stream": {
"posts": [
{
"id": 14,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-02T11:12:29.476Z",
"cooked": "<p>This is a test post.</p>",
"post_number": 1,
"post_type": 1,
"updated_at": "2019-08-02T11:12:29.476Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 15.4,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"read": false,
"user_title": null,
"actions_summary": [
{
"id": 2,
"count": 1,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 16,
"name": "dandelion protocol acct",
"username": "dl-proto",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png",
"created_at": "2019-08-02T11:14:56.285Z",
"cooked": "<p>Ah, an excellent post. I will like it and reply.</p>",
"post_number": 2,
"post_type": 1,
"updated_at": "2019-08-02T11:14:56.285Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 2,
"score": 0.4,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"display_username": "dandelion protocol acct",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"read": false,
"user_title": null,
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": false,
"staff": false,
"user_id": 3,
"hidden": false,
"trust_level": 1,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-08-02T11:14:13.800Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
},
{
"id": 22,
"name": "D",
"username": "d11",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"created_at": "2019-08-04T20:50:39.667Z",
"cooked": "<p>Adding another post, after having deleted a whole thread.</p>",
"post_number": 3,
"post_type": 1,
"updated_at": "2019-08-04T20:50:39.667Z",
"reply_count": 0,
"reply_to_post_number": null,
"quote_count": 0,
"incoming_link_count": 0,
"reads": 1,
"score": 0.2,
"yours": false,
"topic_id": 11,
"topic_slug": "my-first-test-post",
"display_username": "D",
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_bg_color": null,
"primary_group_flair_color": null,
"version": 1,
"can_edit": false,
"can_delete": false,
"can_recover": false,
"can_wiki": false,
"read": false,
"user_title": null,
"actions_summary": [
{
"id": 2,
"can_act": true
},
{
"id": 3,
"can_act": true
},
{
"id": 4,
"can_act": true
},
{
"id": 8,
"can_act": true
},
{
"id": 6,
"can_act": true
},
{
"id": 7,
"can_act": true
}
],
"moderator": false,
"admin": true,
"staff": true,
"user_id": 2,
"hidden": false,
"trust_level": 4,
"deleted_at": null,
"user_deleted": false,
"edit_reason": null,
"can_view_edit_history": true,
"wiki": false,
"user_created_at": "2019-04-02T20:22:14.312Z",
"user_date_of_birth": null,
"can_accept_answer": false,
"can_unaccept_answer": false,
"accepted_answer": false
}
],
"stream": [
14,
16,
22
]
},
"timeline_lookup": [
[
1,
12
],
[
3,
9
]
],
"suggested_topics": [
{
"id": 13,
"title": "A Thread With Links",
"fancy_title": "A Thread With Links",
"slug": "a-thread-with-links",
"posts_count": 2,
"reply_count": 0,
"highest_post_number": 2,
"image_url": null,
"created_at": "2019-08-02T11:15:53.843Z",
"last_posted_at": "2019-08-02T11:16:31.313Z",
"bumped": true,
"bumped_at": "2019-08-02T11:17:12.755Z",
"unseen": false,
"pinned": false,
"unpinned": null,
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 6,
"category_id": 1,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": 3,
"username": "dl-proto",
"name": "dandelion protocol acct",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png"
}
},
{
"extras": null,
"description": "Frequent Poster",
"user": {
"id": 2,
"username": "d11",
"name": "D",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png"
}
}
]
},
{
"id": 7,
"title": "Welcome to Discourse",
"fancy_title": "Welcome to Discourse",
"slug": "welcome-to-discourse",
"posts_count": 1,
"reply_count": 0,
"highest_post_number": 1,
"image_url": "https://sjc3.discourse-cdn.com/free1/images/welcome/discourse-edit-post-animated.gif",
"created_at": "2019-04-02T20:20:12.879Z",
"last_posted_at": "2019-04-02T20:20:12.933Z",
"bumped": true,
"bumped_at": "2019-04-10T23:16:19.399Z",
"unseen": false,
"pinned": true,
"unpinned": null,
"excerpt": "This is a test instance. \nThe first paragraph of this pinned topic will be visible as a welcome message to all new visitors on your homepage. Its important! \nEdit this into a brief description of your community: \n\nWho i&hellip;",
"visible": true,
"closed": false,
"archived": false,
"bookmarked": null,
"liked": null,
"archetype": "regular",
"like_count": 0,
"views": 1,
"category_id": 1,
"featured_link": null,
"has_accepted_answer": false,
"posters": [
{
"extras": "latest single",
"description": "Original Poster, Most Recent Poster",
"user": {
"id": -1,
"username": "system",
"name": "system",
"avatar_template": "/user_avatar/sourcecred-test.discourse.group/system/{size}/1_2.png"
}
}
]
}
],
"id": 11,
"title": "My First Test Post",
"fancy_title": "My First Test Post",
"posts_count": 3,
"created_at": "2019-08-02T11:12:29.408Z",
"views": 8,
"reply_count": 0,
"like_count": 1,
"last_posted_at": "2019-08-04T20:50:39.667Z",
"visible": true,
"closed": false,
"archived": false,
"has_summary": false,
"archetype": "regular",
"slug": "my-first-test-post",
"category_id": 1,
"word_count": 24,
"deleted_at": null,
"user_id": 2,
"featured_link": null,
"pinned_globally": false,
"pinned_at": null,
"pinned_until": null,
"draft": null,
"draft_key": "topic_11",
"draft_sequence": 0,
"unpinned": null,
"pinned": false,
"current_post_number": 1,
"highest_post_number": 3,
"deleted_by": null,
"actions_summary": [
{
"id": 4,
"count": 0,
"hidden": false,
"can_act": true
},
{
"id": 8,
"count": 0,
"hidden": false,
"can_act": true
},
{
"id": 7,
"count": 0,
"hidden": false,
"can_act": true
}
],
"chunk_size": 20,
"bookmarked": null,
"topic_timer": null,
"private_topic_timer": null,
"message_bus_last_id": 0,
"participant_count": 2,
"tags_disable_ads": false,
"details": {
"notification_level": 1,
"can_create_post": true,
"can_reply_as_new_topic": true,
"can_flag_topic": true,
"participants": [
{
"id": 2,
"username": "d11",
"name": "D",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png",
"post_count": 2,
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_color": null,
"primary_group_flair_bg_color": null
},
{
"id": 3,
"username": "dl-proto",
"name": "dandelion protocol acct",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/8797f3/{size}.png",
"post_count": 1,
"primary_group_name": null,
"primary_group_flair_url": null,
"primary_group_flair_color": null,
"primary_group_flair_bg_color": null
}
],
"created_by": {
"id": 2,
"username": "d11",
"name": "D",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png"
},
"last_poster": {
"id": 2,
"username": "d11",
"name": "D",
"avatar_template": "https://avatars.discourse.org/v4/letter/d/47e85d/{size}.png"
}
}
}

View File

@ -0,0 +1,39 @@
#!/bin/bash
set -eu
snapshots_dir=src/plugins/discourse/snapshots
test_instance_url="https://sourcecred-test.discourse.group"
test_instance_username="credbot"
if [ -z "${DISCOURSE_TEST_API_KEY:-}" ]; then
printf >&2 'Please set the DISCOURSE_TEST_API_KEY environment variable.\n'
printf >&2 'Contact the SourceCred maintainers to get the key.\n'
exit 1
fi
if [ ! "$(jq --version)" ]; then
printf >&2 'This script depends on jq. Please install it.\n'
exit 1
fi
toplevel="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)"
cd "${toplevel}"
fetch() {
url="${test_instance_url}$1"
filename="$(printf '%s' "${url}" | base64 -w 0 | tr -d '=' | tr '/+' '_-')"
path="${snapshots_dir}/${filename}"
curl -sfL "$url" \
-H "Api-Key: ${DISCOURSE_TEST_API_KEY}" \
-H "Api-Username: ${test_instance_username}" \
-H "Accept: application/json" \
| jq '.' > "${path}"
}
rm -r "${snapshots_dir}"
mkdir "${snapshots_dir}"
fetch "/latest.json?order=created"
fetch "/posts.json"
fetch "/t/11.json"
fetch "/posts/14.json"