Implement a command-line interface (#217)

Summary:
This commit implements the `sourcecred` command-line utility, which has
three subcommands:
  - `plugin-graph` creates one plugin’s graph;
  - `combine` combines multiple on-disk graphs; and
  - `graph` creates all plugins’ graphs and combines them.

As an implementation detail, the `into.sh` script is very convenient,
avoiding needing to do any pipe management in Node (which is Not Fun).
When we build for release, we may want to factor that differently.

Test Plan:
To see it all in action, run `yarn backend`, and then try:
```
$ export SOURCECRED_GITHUB_TOKEN="your_token_here"
$ node ./bin/sourcecred.js graph sourcecred sourcecred
Using output directory: /tmp/sourcecred/sourcecred

Starting tasks
  GO   create-git
  GO   create-github
 PASS  create-github
 PASS  create-git
  GO   combine
 PASS  combine

Full results
 PASS  create-git
 PASS  create-github
 PASS  combine

Overview
Final result:  SUCCESS

$ ls /tmp/sourcecred/sourcecred/
graph-github.json  graph-git.json  graph.json

$ jq '.nodes | length' /tmp/sourcecred/sourcecred/*.json
1000
7302
8302
```
The `node sourcecred.js graph` command takes 9.8s for me.

(The salient point of the last command is that the two small graphs have
node count adding up to the node count of the big graph. Incidentally,
we are [almost][1] at a nice round number of nodes in the GitHub graph.)

[1]: https://xkcd.com/1000/

wchargin-branch: cli

config/paths.js

@@ -57,8 +57,9 @@ module.exports = {
   // source file, and the key will be the filename of the bundled entry
   // point within the build directory.
   backendEntryPoints: {
-    "commands/goodbye": resolveApp("src/cli/commands/goodbye.js"),
-    "commands/example": resolveApp("src/cli/commands/example.js"),
+    "commands/combine": resolveApp("src/cli/commands/combine.js"),
+    "commands/graph": resolveApp("src/cli/commands/graph.js"),
+    "commands/plugin-graph": resolveApp("src/cli/commands/pluginGraph.js"),
     sourcecred: resolveApp("src/cli/sourcecred.js"),
     fetchAndPrintGithubRepo: resolveApp(
       "src/plugins/github/bin/fetchAndPrintGithubRepo.js"

src/cli/commands/combine.js

@@ -0,0 +1,41 @@
+// @flow
+
+import {Command} from "@oclif/command";
+import fs from "fs";
+import stringify from "json-stable-stringify";
+import {promisify} from "util";
+
+import {Graph} from "../../core/graph";
+
+export default class CombineCommand extends Command {
+  static description = "combine multiple contribution graphs into one big graph";
+
+  // We disable strict-mode so that we can accept a variable number of
+  // arguments. Although this is what the docs [1] suggest, it doesn't
+  // seem like the best way to do it. TODO: Investigate.
+  //
+  // [1]: https://oclif.io/docs/commands.html
+  static strict = false;
+
+  static usage = "" +
+    "combine [GRAPH]...\n" +
+    "  where each GRAPH is a JSON file generated by plugin-graph";
+
+  async run() {
+    const {argv} = this.parse(CombineCommand);
+    combine(argv);
+  }
+}
+
+async function combine(filenames: $ReadOnlyArray<string>) {
+  const readFile = promisify(fs.readFile);
+  const graphs = await Promise.all(
+    filenames.map((filename) =>
+      readFile(filename).then((contents) =>
+        Graph.fromJSON(JSON.parse(contents))
+      )
+    )
+  );
+  const result = Graph.mergeManyConservative(graphs);
+  console.log(stringify(result, {space: 4}));
+}

src/cli/commands/example.js

@@ -1,14 +0,0 @@
-// @flow
-import {Command, flags} from "@oclif/command";
-
-export default class ExampleCommand extends Command {
-  static description = "An example command description";
-  static flags = {
-    name: flags.string({char: "n", description: "name to print"}),
-  };
-  async run() {
-    const {flags} = this.parse(ExampleCommand);
-    const name = flags.name || "world";
-    this.log(`hello ${name} EXAMPEL`);
-  }
-}

src/cli/commands/goodbye.js

@@ -1,14 +0,0 @@
-// @flow
-import {Command, flags} from "@oclif/command";
-
-export default class GoodbyeCommand extends Command {
-  static description = "Another example command description";
-  static flags = {
-    name: flags.string({char: "n", description: "name to print"}),
-  };
-  async run() {
-    const {flags} = this.parse(GoodbyeCommand);
-    const name = flags.name || "world";
-    this.log(`goodbye ${name}`);
-  }
-}

src/cli/commands/graph.js

@@ -0,0 +1,111 @@
+// @flow
+
+import {Command, flags} from "@oclif/command";
+import mkdirp from "mkdirp";
+import os from "os";
+import path from "path";
+
+import {pluginNames} from "../common";
+
+const execDependencyGraph = require("../../tools/execDependencyGraph").default;
+
+export default class GraphCommand extends Command {
+  static description = `\
+create the contribution graph for a repository
+
+Create the contribution graph for a repository. This creates a
+contribution graph for each individual plugin, and then combines the
+individual graphs into one larger graph. The graphs are stored as JSON
+files under OUTPUT_DIR/REPO_OWNER/REPO_NAME, where OUTPUT_DIR is
+configurable.
+`.trim();
+
+  static args = [
+    {
+      name: "repo_owner",
+      required: true,
+      description: "owner of the GitHub repository for which to fetch data",
+    },
+    {
+      name: "repo_name",
+      required: true,
+      description: "name of the GitHub repository for which to fetch data",
+    },
+  ];
+
+  static flags = {
+    "output-directory": flags.string({
+      short: "o",
+      description: "directory into which to store graphs",
+      env: "SOURCECRED_OUTPUT_DIRECTORY",
+      default: path.join(os.tmpdir(), "sourcecred"),
+    }),
+    "github-token": flags.string({
+      description:
+        "a GitHub API token, as generated at " +
+        "https://github.com/settings/tokens/new",
+      required: true,
+      env: "SOURCECRED_GITHUB_TOKEN",
+    }),
+  };
+
+  async run() {
+    const {
+      args: {repo_owner: repoOwner, repo_name: repoName},
+      flags: {"github-token": token, "output-directory": outputDirectory},
+    } = this.parse(GraphCommand);
+    graph(outputDirectory, repoOwner, repoName, token);
+  }
+}
+
+function graph(
+  outputDirectory: string,
+  repoOwner: string,
+  repoName: string,
+  token: string
+) {
+  const scopedDirectory = path.join(outputDirectory, repoOwner, repoName);
+  console.log("Storing graphs into: " + scopedDirectory);
+  mkdirp.sync(scopedDirectory);
+  const tasks = makeTasks(scopedDirectory, {repoOwner, repoName, token});
+  execDependencyGraph(tasks).then(({success}) => {
+    process.exitCode = success ? 0 : 1;
+  });
+}
+
+function makeTasks(outputDirectory, {repoOwner, repoName, token}) {
+  const taskId = (id) => `create-${id}`;
+  const graphFilename = (id) => path.join(outputDirectory, `graph-${id}.json`);
+  const into = "./src/cli/into.sh";
+  return [
+    ...pluginNames().map((id) => ({
+      id: taskId(id),
+      cmd: [
+        into,
+        graphFilename(id),
+        "node",
+        "./bin/sourcecred.js",
+        "plugin-graph",
+        "--plugin",
+        id,
+        repoOwner,
+        repoName,
+        "--github-token",
+        token,
+      ],
+      deps: [],
+    })),
+    {
+      id: "combine",
+      cmd: [
+        into,
+        path.join(outputDirectory, "graph.json"),
+        "node",
+        "./bin/sourcecred.js",
+        "combine",
+        ...pluginNames().map((id) => graphFilename(id)),
+      ],
+      deps: pluginNames().map((id) => taskId(id)),
+    },
+  ];
+}

src/cli/commands/pluginGraph.js

@@ -0,0 +1,87 @@
+// @flow
+
+import {Command, flags} from "@oclif/command";
+import stringify from "json-stable-stringify";
+
+import type {Graph} from "../../core/graph";
+import type {PluginName} from "../common";
+import createGitGraph from "../../plugins/git/cloneGitGraph";
+import createGithubGraph from "../../plugins/github/fetchGithubGraph";
+import {pluginNames} from "../common";
+
+export default class PluginGraphCommand extends Command {
+  static description = "create the contribution graph for a single plugin";
+
+  static args = [
+    {
+      name: "repo_owner",
+      required: true,
+      description: "owner of the GitHub repository for which to fetch data",
+    },
+    {
+      name: "repo_name",
+      required: true,
+      description: "name of the GitHub repository for which to fetch data",
+    },
+  ];
+
+  static flags = {
+    plugin: flags.string({
+      description: "plugin whose graph to generate",
+      required: true,
+      options: pluginNames(),
+    }),
+    "github-token": flags.string({
+      description:
+        "a GitHub API token, as generated at " +
+        "https://github.com/settings/tokens/new" +
+        "; required only if using the GitHub plugin",
+      env: "SOURCECRED_GITHUB_TOKEN",
+    }),
+  };
+
+  async run() {
+    const {
+      args: {repo_owner: repoOwner, repo_name: repoName},
+      flags: {"github-token": token, plugin},
+    } = this.parse(PluginGraphCommand);
+    pluginGraph(plugin, repoOwner, repoName, token);
+  }
+}
+
+function pluginGraph(
+  plugin: PluginName,
+  repoOwner: string,
+  repoName: string,
+  githubToken?: string
+) {
+  switch (plugin) {
+    case "git":
+      display(Promise.resolve(createGitGraph(repoOwner, repoName)));
+      break;
+    case "github":
+      if (githubToken == null) {
+        // TODO: This check should be abstracted so that plugins can
+        // specify their argument dependencies and get nicely
+        // formatted errors.
+        console.error("fatal: No GitHub token specified. Try `--help'.");
+        process.exitCode = 1;
+        return;
+      } else {
+        display(createGithubGraph(repoOwner, repoName, githubToken));
+      }
+      break;
+    default:
+      // eslint-disable-next-line no-unused-expressions
+      (plugin: empty);
+      console.error("fatal: Unknown plugin: " + plugin);
+      process.exitCode = 1;
+      return;
+  }
+}
+
+function display<NP, EP>(promise: Promise<Graph<NP, EP>>) {
+  promise.then((graph) => {
+    console.log(stringify(graph, {space: 4}));
+  });
+}

src/cli/common.js

@@ -0,0 +1,6 @@
+// @flow
+
+export type PluginName = "git" | "github";
+export function pluginNames() {
+  return ["git", "github"];
+}

src/cli/into.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Utility for redirection.
+set -eu
+if [ $# -eq 0 ]; then
+    printf >&2 'into: fatal: no target provided\n'
+    exit 1
+fi
+target="$1"
+shift
+exec "$@" >"${target}"