From e7b1fbd681576defe2086c238df409fbdfc96148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Sun, 18 Aug 2019 16:27:11 +0200 Subject: [PATCH] mirror: allow fetching all usernames (#1293) This is a minor change to the Discourse mirror so that it supports a query to get all users from the server. It will be convenient for a followon change which makes `update` search for every user's likes. I also modified createGraph so that it uses the new method, which results in code that is cleaner and slightly more efficient. Test plan: Unit tests updated. --- src/plugins/discourse/createGraph.js | 6 +++-- src/plugins/discourse/createGraph.test.js | 10 ++++++++ src/plugins/discourse/mirror.js | 29 ++++++++++++++++++++--- src/plugins/discourse/mirror.test.js | 15 ++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/plugins/discourse/createGraph.js b/src/plugins/discourse/createGraph.js index 86796be..310395a 100644 --- a/src/plugins/discourse/createGraph.js +++ b/src/plugins/discourse/createGraph.js @@ -130,17 +130,19 @@ export function createGraph(serverUrl: string, data: DiscourseData): Graph { const g = new Graph(); const topicIdToTitle: Map = new Map(); + for (const username of data.users()) { + g.addNode(userNode(serverUrl, username)); + } + for (const topic of data.topics()) { topicIdToTitle.set(topic.id, topic.title); g.addNode(topicNode(serverUrl, topic)); - g.addNode(userNode(serverUrl, topic.authorUsername)); g.addEdge(authorsTopicEdge(serverUrl, topic)); } for (const post of data.posts()) { const topicTitle = topicIdToTitle.get(post.topicId) || "[unknown topic]"; g.addNode(postNode(serverUrl, post, topicTitle)); - g.addNode(userNode(serverUrl, post.authorUsername)); g.addEdge(authorsPostEdge(serverUrl, post)); g.addEdge(topicContainsPostEdge(serverUrl, post)); let replyToPostIndex = post.replyToPostIndex; diff --git a/src/plugins/discourse/createGraph.test.js b/src/plugins/discourse/createGraph.test.js index 3a07450..f325f74 100644 --- a/src/plugins/discourse/createGraph.test.js +++ b/src/plugins/discourse/createGraph.test.js @@ -40,6 +40,16 @@ describe("plugins/discourse/createGraph", () => { posts(): $ReadOnlyArray { return this._posts; } + users(): $ReadOnlyArray { + const users = new Set(); + for (const {authorUsername} of this.posts()) { + users.add(authorUsername); + } + for (const {authorUsername} of this.topics()) { + users.add(authorUsername); + } + return Array.from(users); + } findPostInTopic(topicId: TopicId, indexWithinTopic: number): ?PostId { const post = this._posts.filter( (p) => p.topicId === topicId && p.indexWithinTopic === indexWithinTopic diff --git a/src/plugins/discourse/mirror.js b/src/plugins/discourse/mirror.js index 2f71bf5..13a08a4 100644 --- a/src/plugins/discourse/mirror.js +++ b/src/plugins/discourse/mirror.js @@ -13,7 +13,7 @@ import { // The version should be bumped any time the database schema is changed, // so that the cache will be properly invalidated. -const VERSION = "discourse_mirror_v1"; +const VERSION = "discourse_mirror_v2"; /** * An interface for retrieving all of the Discourse data at once. @@ -45,6 +45,13 @@ export interface DiscourseData { * Returns undefined if no such post is available. */ findPostInTopic(topicId: TopicId, indexWithinTopic: number): ?PostId; + + /** + * Get usernames for all users. + * + * The order is unspecified. + */ + users(): $ReadOnlyArray; } /** @@ -131,12 +138,14 @@ export class Mirror implements DiscourseData { db.prepare("INSERT INTO meta (zero, config) VALUES (0, ?)").run(config); const tables = [ + "CREATE TABLE users (username TEXT PRIMARY KEY)", dedent`\ CREATE TABLE topics ( id INTEGER PRIMARY KEY, title TEXT NOT NULL, timestamp_ms INTEGER NOT NULL, - author_username TEXT NOT NULL + author_username TEXT NOT NULL, + FOREIGN KEY(author_username) REFERENCES users(username) ) `, dedent`\ @@ -147,7 +156,8 @@ export class Mirror implements DiscourseData { topic_id INTEGER NOT NULL, index_within_topic INTEGER NOT NULL, reply_to_post_index INTEGER, - FOREIGN KEY(topic_id) REFERENCES topics(id) + FOREIGN KEY(topic_id) REFERENCES topics(id), + FOREIGN KEY(author_username) REFERENCES users(username) ) `, ]; @@ -200,6 +210,13 @@ export class Mirror implements DiscourseData { })); } + users(): $ReadOnlyArray { + return this._db + .prepare("SELECT username FROM users") + .pluck() + .all(); + } + findPostInTopic(topicId: TopicId, indexWithinTopic: number): ?PostId { return this._db .prepare( @@ -237,6 +254,9 @@ export class Mirror implements DiscourseData { topicId, authorUsername, } = post; + db.prepare("INSERT OR IGNORE INTO users (username) VALUES (?)").run( + authorUsername + ); db.prepare( dedent`\ REPLACE INTO posts ( @@ -275,6 +295,9 @@ export class Mirror implements DiscourseData { if (topicWithPosts != null) { const {topic, posts} = topicWithPosts; const {id, title, timestampMs, authorUsername} = topic; + db.prepare("INSERT OR IGNORE INTO users (username) VALUES (?)").run( + authorUsername + ); this._db .prepare( dedent`\ diff --git a/src/plugins/discourse/mirror.test.js b/src/plugins/discourse/mirror.test.js index c112ab9..9531b6d 100644 --- a/src/plugins/discourse/mirror.test.js +++ b/src/plugins/discourse/mirror.test.js @@ -194,6 +194,21 @@ describe("plugins/discourse/mirror", () => { expect(mirror.posts()).toEqual(posts); }); + it("provides usernames for all active users", async () => { + const {mirror, fetcher} = example(); + fetcher.addPost(2, null, "alpha"); + fetcher.addPost(3, null, "beta"); + fetcher.addPost(3, 1, "alpha"); + await mirror.update(); + // credbot appears because it is the nominal author of all topics + expect( + mirror + .users() + .slice() + .sort() + ).toEqual(["alpha", "beta", "credbot"]); + }); + describe("update semantics", () => { it("only fetches new topics on `update`", async () => { const {mirror, fetcher} = example();