cli2: add Discourse CLI plugin (#1814)

This updates the v2 CLI so that it now supports the Discourse plugin.

Test plan:

Modify the test instance described in the previous commits so that the
root `sourcecred.json` file includes "sourcecred/discourse" in the list
of bundled plugins. Then, add a
`config/sourcecred/discourse/config.json` file with the following
contents:
`{"serverUrl": "https://sourcecred-test.discourse.group/"}`

Now, running `sc2 load` will load Discourse data, and `sc2 graph` writes
a Discourse graph in the output directory.

src/cli2/bundledPlugins.js

@@ -2,11 +2,15 @@
 
 import type {CliPlugin} from "./cliPlugin";
 import {GithubCliPlugin} from "../plugins/github/cliPlugin";
+import {DiscourseCliPlugin} from "../plugins/discourse/cliPlugin";
 
 /**
  * Returns an object mapping owner-name pairs to CLI plugin
  * declarations; keys are like `sourcecred/github`.
  */
 export function bundledPlugins(): {[pluginId: string]: CliPlugin} {
-  return {"sourcecred/github": new GithubCliPlugin()};
+  return {
+    "sourcecred/github": new GithubCliPlugin(),
+    "sourcecred/discourse": new DiscourseCliPlugin(),
+  };
 }

src/plugins/discourse/cliPlugin.js

@@ -0,0 +1,74 @@
+// @flow
+
+import Database from "better-sqlite3";
+import fs from "fs-extra";
+import {join as pathJoin} from "path";
+
+import type {CliPlugin, PluginDirectoryContext} from "../../cli2/cliPlugin";
+import type {PluginDeclaration} from "../../analysis/pluginDeclaration";
+import type {ReferenceDetector} from "../../core/references/referenceDetector";
+import type {WeightedGraph} from "../../core/weightedGraph";
+import {createGraph} from "./createGraph";
+import {declaration} from "./declaration";
+import {parseConfig, type DiscourseConfig} from "./config";
+import {weightsForDeclaration} from "../../analysis/pluginDeclaration";
+import {SqliteMirrorRepository} from "./mirrorRepository";
+import {Fetcher} from "./fetch";
+import {Mirror} from "./mirror";
+import {DiscourseReferenceDetector} from "./referenceDetector";
+import {type TaskReporter} from "../../util/taskReporter";
+
+async function loadConfig(
+  dirContext: PluginDirectoryContext
+): Promise<DiscourseConfig> {
+  const dirname = dirContext.configDirectory();
+  const path = pathJoin(dirname, "config.json");
+  const contents = await fs.readFile(path);
+  return Promise.resolve(parseConfig(JSON.parse(contents)));
+}
+
+async function repository(
+  ctx: PluginDirectoryContext,
+  serverUrl: string
+): Promise<SqliteMirrorRepository> {
+  const path = pathJoin(ctx.cacheDirectory(), "discourseMirror.db");
+  const db = await new Database(path);
+  return new SqliteMirrorRepository(db, serverUrl);
+}
+
+export class DiscourseCliPlugin implements CliPlugin {
+  declaration(): PluginDeclaration {
+    return declaration;
+  }
+
+  async load(
+    ctx: PluginDirectoryContext,
+    reporter: TaskReporter
+  ): Promise<void> {
+    const {serverUrl, mirrorOptions} = await loadConfig(ctx);
+    const repo = await repository(ctx, serverUrl);
+    const fetcher = new Fetcher({serverUrl});
+    const mirror = new Mirror(repo, fetcher, serverUrl, mirrorOptions);
+    await mirror.update(reporter);
+  }
+
+  async graph(
+    ctx: PluginDirectoryContext,
+    rd: ReferenceDetector
+  ): Promise<WeightedGraph> {
+    const _ = rd; // TODO(#1808): not yet used
+    const config = await loadConfig(ctx);
+    const repo = await repository(ctx, config.serverUrl);
+    const graph = createGraph(config.serverUrl, repo);
+    const weights = weightsForDeclaration(declaration);
+    return {graph, weights};
+  }
+
+  async referenceDetector(
+    ctx: PluginDirectoryContext
+  ): Promise<ReferenceDetector> {
+    const config = await loadConfig(ctx);
+    const repo = await repository(ctx, config.serverUrl);
+    return new DiscourseReferenceDetector(repo);
+  }
+}

src/plugins/discourse/config.js

@@ -0,0 +1,67 @@
+// @flow
+
+import {type MirrorOptions} from "./mirror";
+
+export type DiscourseConfig = {|
+  +serverUrl: string,
+  +mirrorOptions?: $Shape<MirrorOptions>,
+|};
+
+type JsonObject =
+  | string
+  | number
+  | boolean
+  | null
+  | JsonObject[]
+  | {[string]: JsonObject};
+
+export function parseConfig(raw: JsonObject): DiscourseConfig {
+  if (raw == null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error("bad config: " + JSON.stringify(raw));
+  }
+  let mirrorOptions = undefined;
+  const {serverUrl} = raw;
+  if (typeof serverUrl !== "string") {
+    throw new Error("serverUrl not string: " + JSON.stringify(serverUrl));
+  }
+  const httpRE = new RegExp(/^https?:\/\//);
+  if (!httpRE.test(serverUrl)) {
+    throw new Error(
+      "expected server url to start with 'https://' or 'http://'"
+    );
+  }
+  if (raw.mirrorOptions !== undefined) {
+    const {mirrorOptions: rawMO} = raw;
+    if (rawMO == null || typeof rawMO !== "object" || Array.isArray(rawMO)) {
+      throw new Error("bad config: " + JSON.stringify(rawMO));
+    }
+    const {recheckCategoryDefinitionsAfterMs} = rawMO;
+    let {recheckTopicsInCategories} = rawMO;
+
+    if (!Array.isArray(recheckTopicsInCategories)) {
+      throw new Error(
+        "mirrorOptions.recheckTopicsInCategories must be array, got " +
+          JSON.stringify(recheckTopicsInCategories)
+      );
+    }
+    if (!recheckTopicsInCategories.every((x) => typeof x === "number")) {
+      throw new Error(
+        "mirrorOptions.recheckTopicsInCategories must all be numbers, got " +
+          JSON.stringify(recheckTopicsInCategories)
+      );
+    }
+    recheckTopicsInCategories = recheckTopicsInCategories.map((x) => Number(x));
+    if (typeof recheckCategoryDefinitionsAfterMs !== "number") {
+      throw new Error(
+        "recheckCategoryDefinitionsAfterMs must be number, got " +
+          JSON.stringify(recheckCategoryDefinitionsAfterMs)
+      );
+    }
+    mirrorOptions = {
+      recheckCategoryDefinitionsAfterMs,
+      recheckTopicsInCategories,
+    };
+  }
+
+  return {serverUrl, mirrorOptions};
+}

src/plugins/discourse/config.test.js

@@ -0,0 +1,72 @@
+// @flow
+
+import {parseConfig} from "./config";
+
+describe("plugins/discourse/config", () => {
+  describe("parseConfig", () => {
+    it("works on a config with just a serverUrl", () => {
+      const config = {serverUrl: "https://server.io"};
+      expect(parseConfig(config)).toEqual(config);
+    });
+    it("errors if the serverUrl is not a url", () => {
+      const config = {serverUrl: "1234"};
+      expect(() => parseConfig(config)).toThrowError();
+    });
+    it("errors if the serverUrl is not a string", () => {
+      const config = {serverUrl: 234};
+      expect(() => parseConfig(config)).toThrowError();
+    });
+    it("errors if the serverUrl is missing", () => {
+      const config = {};
+      expect(() => parseConfig(config)).toThrowError();
+    });
+    it("errors if the config is not an object", () => {
+      const config = [];
+      expect(() => parseConfig(config)).toThrowError();
+    });
+    it("works on a config with mirror options", () => {
+      const config = {
+        serverUrl: "https://server.io",
+        mirrorOptions: {
+          recheckCategoryDefinitionsAfterMs: 12,
+          recheckTopicsInCategories: [1, 2, 3, 4],
+        },
+      };
+      expect(parseConfig(config)).toEqual(config);
+    });
+    it("errors on a config with missing mirror options", () => {
+      const c1 = {
+        serverUrl: "https://server.io",
+        mirrorOptions: {
+          recheckTopicsInCategories: [1, 2, 3, 4],
+        },
+      };
+      expect(() => parseConfig(c1)).toThrowError();
+      const c2 = {
+        serverUrl: "https://server.io",
+        mirrorOptions: {
+          recheckCategoryDefinitionsAfterMs: 12,
+        },
+      };
+      expect(() => parseConfig(c2)).toThrowError();
+    });
+    it("errors with bad config options", () => {
+      const c1 = {
+        serverUrl: "https://server.io",
+        mirrorOptions: {
+          recheckTopicsInCategories: 12,
+          recheckCategoryDefinitionsAfterMs: 12,
+        },
+      };
+      expect(() => parseConfig(c1)).toThrowError();
+      const c2 = {
+        serverUrl: "https://server.io",
+        mirrorOptions: {
+          recheckTopicsInCategories: [1, 2, 3, 4],
+          recheckCategoryDefinitionsAfterMs: "foo",
+        },
+      };
+      expect(() => parseConfig(c2)).toThrowError();
+    });
+  });
+});