Add `sourcecred discourse` command (#1374)

This adds a new command, `discourse`, which makes it convenient to load
Discourse servers as standalone SourceCred projects.

For example, you could load the official SourceCred discourse via the
following:

```sh
export SOURCECRED_DISCOURSE_KEY=....
yarn backend
node bin/sourcecred.js discourse https://discourse.sourcecred.io credbot
yarn start
```

I've updated the README with instructions for using the plugin.

Test plan: No automated testing because I see this tool as a temporary
placeholder until we get the SourceCred instances setup. I manually
tested the error cases (e.g. providing an invalid server url) as well as
success cases like the one above. I validated that the weights file
argument is being interpreted correctly (i.e. trying to load invalid
weights produces an expected error message, loading valid weights
results in those weights being present in the UI).

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 
 <!-- Please add new entries just beneath this line. -->
+- Add `sourcecred discourse` for loading Discourse servers (#1374)
 - Breaking: Change output format for the scores command (#1372)
 - Include top nodes for every type in Timeline Cred (#1358)
 

README.md

@@ -59,6 +59,29 @@ yarn start
 
 Finally, we can navigate a browser window to `localhost:8080` to view generated data.
 
+### Loading a Discourse Server
+
+SourceCred can also run on Discourse instances!
+
+To do so, you'll first need admin access on the Discourse server in question. Generate
+an admin API key, available at the `/admin/api/keys`. You should also create a user account
+on the instance that will be the nominal user for the API requests. You shouldn't use an admin
+user identity for this, because then SourceCred could pick up private or deleted posts. Instead,
+we recommend making a user called "credbot" with no special permissions.
+
+Once you have the key and user ready, prepare SourceCred using the same steps as above,
+and then use the `sourcecred discourse` command, providing the server url, and then the username.
+Below is an example for loading the cred for SourceCred's [own discourse instance][forum].
+
+```Bash
+git clone https://github.com/sourcecred/sourcecred.git
+cd sourcecred
+yarn install
+yarn backend
+export SOURCECRED_DISCOURSE_KEY=$YOUR_KEY
+node bin/sourcecred.js discourse https://discourse.sourcecred.io credbot
+```
+
 ### Running with Docker
 
 You can build and run sourcecred in a container to avoid installing dependencies on your host. First, build the container:
@@ -74,7 +97,7 @@ $ docker build --build-arg SOURCECRED_DEFAULT_DIRECTORY=/tmp/data \
   -t sourcecred/sourcecred .
 ```
 
-Your options for running the container including the following commands. 
+Your options for running the container including the following commands.
 Examples will be shown for each.
 
  - **dev-preview**: offers a shortcut for loading sourcecred and then starting a dev server. This is likely the option you'll choose if you want to provide a respository or an organization and preview results a web interface.
@@ -82,10 +105,10 @@ Examples will be shown for each.
  - **build**: simply provides the build command to yarn, followed by any argumnents that you provide.
  - **(anything else)**: will be passed on to sourcecred.js
 
-#### Development Preview 
+#### Development Preview
 
-To run the development preview, you will still need to export a GitHub token, and then provide it to the container when you run it. 
-Notice that we are also binding port 8080 so we can view the web interface that will be opened up.  
+To run the development preview, you will still need to export a GitHub token, and then provide it to the container when you run it.
+Notice that we are also binding port 8080 so we can view the web interface that will be opened up.
 The only argument needed is a command to load the GitHub repository to generate the sourcecred for:
 
 ```bash
@@ -104,7 +127,7 @@ $ SOURCECRED_GITHUB_TOKEN="xxxxxxxxxxxxxxxxx" \
   -p 8080:8080 sourcecred/sourcecred dev-preview "${ORGANIZATION}"
 ```
 
-If you want to bind the data folder to the host, you can do that too. 
+If you want to bind the data folder to the host, you can do that too.
 In the example below, we have a folder "data" in the present working directory that we bind to "/data" in the container, the default `SOURCECRED_DIRECTORY`. We can then generate the data (and it will
 be saved there):
 
@@ -182,7 +205,7 @@ When you are finished, stop and remove the container.
 $ docker stop sourcecred
 ```
 
-Since we used the remove (--rm) tag, stopping it will also remove it. 
+Since we used the remove (--rm) tag, stopping it will also remove it.
 If you bound the data folder to the host, you'll see the output remaining there from the generation:
 
 ```bash

src/cli/discourse.js

@@ -0,0 +1,140 @@
+// @flow
+// Implementation of `sourcecred discourse`
+// This is a (likely temporary command) to facilitate loading a single
+// discourse server.
+
+import dedent from "../util/dedent";
+import {LoggingTaskReporter} from "../util/taskReporter";
+import type {Command} from "./command";
+import * as Common from "./common";
+import {defaultWeights} from "../analysis/weights";
+import {load} from "../api/load";
+import {declaration as discourseDeclaration} from "../plugins/discourse/declaration";
+import {type Project} from "../core/project";
+
+function usage(print: (string) => void): void {
+  print(
+    dedent`\
+    usage: sourcecred discourse DISCOURSE_URL DISCOURSE_USERNAME
+                                [--weights WEIGHTS_FILE]
+           sourcecred discourse --help
+
+    Loads a target Discourse server, generating cred scores for it.
+
+    Arguments:
+        DISCOURSE_URL
+            The url to the Discourse server in question, for example
+            https://discourse.sourcecred.io
+
+        DISCOURSE_USERNAME
+            A user account on the Discourse server, to be used as the
+            "perspective" for the Discourse API calls. This user should not be
+            a privileged or admin user, otherwise hidden or deleted topics may
+            be included in the results. We recommend making a new user called
+            "credbot" on the server, with no special roles or permissions.
+
+        --weights WEIGHTS_FILE
+            Path to a json file which contains a weights configuration.
+            This will be used instead of the default weights and persisted.
+
+        --help
+            Show this help message and exit, as 'sourcecred help discourse'.
+
+    Environment variables:
+        SOURCECRED_DISCOURSE_KEY
+            A Discourse admin API key generated from discourse server in
+            question.
+
+            To generate a key, use the /admin/api/keys route on your Discourse
+            server, e.g. https://discourse.example.com/admin/api/keys
+
+        SOURCECRED_DIRECTORY
+            Directory owned by SourceCred, in which data, caches,
+            registries, etc. are stored. Optional: defaults to a
+            directory 'sourcecred' under your OS's temporary directory;
+            namely:
+                ${Common.defaultSourcecredDirectory()}
+    `.trimRight()
+  );
+}
+
+function die(std, message) {
+  std.err("fatal: " + message);
+  std.err("fatal: run 'sourcecred help discourse' for help");
+  return 1;
+}
+
+const command: Command = async (args, std) => {
+  const positionalArgs = [];
+  let weightsPath: string | null = null;
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case "--help": {
+        usage(std.out);
+        return 0;
+      }
+      case "--weights": {
+        if (weightsPath != undefined)
+          return die(std, "'--weights' given multiple times");
+        if (++i >= args.length)
+          return die(std, "'--weights' given without value");
+        weightsPath = args[i];
+        break;
+      }
+      default: {
+        positionalArgs.push(args[i]);
+        break;
+      }
+    }
+  }
+  if (positionalArgs.length !== 2) {
+    return die(std, "Expected two positional arguments (or --help).");
+  }
+  const [serverUrl, apiUsername] = positionalArgs;
+  let projectId = serverUrl;
+  if (projectId.startsWith("https://")) {
+    projectId = projectId.slice("https://".length);
+  } else if (projectId.startsWith("http://")) {
+    projectId = projectId.slice("http://".length);
+  } else {
+    die(std, "expected server url to start with 'https://' or 'http://'");
+  }
+
+  const project: Project = {
+    id: projectId,
+    repoIds: [],
+    discourseServer: {serverUrl, apiUsername},
+  };
+  const taskReporter = new LoggingTaskReporter();
+  let weights = defaultWeights();
+  if (weightsPath) {
+    weights = await Common.loadWeights(weightsPath);
+  }
+  const plugins = [discourseDeclaration];
+
+  await load(
+    {
+      project,
+      params: {weights},
+      plugins,
+      sourcecredDirectory: Common.sourcecredDirectory(),
+      githubToken: null,
+      discourseKey: Common.discourseKey(),
+    },
+    taskReporter
+  );
+  return 0;
+};
+
+export const help: Command = async (args, std) => {
+  if (args.length === 0) {
+    usage(std.out);
+    return 0;
+  } else {
+    usage(std.err);
+    return 1;
+  }
+};
+
+export default command;

src/cli/help.js

@@ -8,6 +8,7 @@ import {help as loadHelp} from "./load";
 import {help as scoresHelp} from "./scores";
 import {help as clearHelp} from "./clear";
 import {help as genProjectHelp} from "./genProject";
+import {help as discourseHelp} from "./discourse";
 
 const help: Command = async (args, std) => {
   if (args.length === 0) {
@@ -21,6 +22,7 @@ const help: Command = async (args, std) => {
     scores: scoresHelp,
     clear: clearHelp,
     "gen-project": genProjectHelp,
+    discourse: discourseHelp,
   };
   if (subHelps[command] !== undefined) {
     return subHelps[command](args.slice(1), std);
@@ -43,6 +45,7 @@ function usage(print: (string) => void): void {
       clear         clear SoucrceCred data
       scores        print SourceCred scores to stdout
       gen-project   print a SourceCred project config to stdout
+      discourse     load a Discourse server into SourceCred
       help          show this help message
 
     Use 'sourcecred help COMMAND' for help about an individual command.

src/cli/sourcecred.js

@@ -10,6 +10,7 @@ import load from "./load";
 import scores from "./scores";
 import clear from "./clear";
 import genProject from "./genProject";
+import discourse from "./discourse";
 
 const sourcecred: Command = async (args, std) => {
   if (args.length === 0) {
@@ -31,6 +32,8 @@ const sourcecred: Command = async (args, std) => {
       return scores(args.slice(1), std);
     case "gen-project":
       return genProject(args.slice(1), std);
+    case "discourse":
+      return discourse(args.slice(1), std);
     default:
       std.err("fatal: unknown command: " + JSON.stringify(args[0]));
       std.err("fatal: run 'sourcecred help' for commands and usage");