Add CLI command: `sourcecred export-graph` (#1110)

* Add CLI command: `sourcecred export-graph`

This adds an `export-graph` command to the SourceCred CLI. It exports
the combined cred graphs for individual RepoIds, as was done for
[sourcecred/research#4].

Example usage:
```
$ node bin/sourcecred.js load sourcecred/mission
$ node bin/sourcecred.js export-graph sourcecred/mission >
  /tmp/mission_graph.json
```

Test plan:
The new command is thoroughly unit tested.

[sourcecred/research#4]: https://github.com/sourcecred/research/pull/4

* Address review feedback by @wchargin

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## [Unreleased]
+- Add the `export-graph` command (#1110)
 - Enable loading private repositories (#1085)
 - Enable setting type weights to 0 in the UI (#1005)
 - Add support for 🚀 and 👀 reaction types (#1068)

src/cli/exportGraph.js

@@ -0,0 +1,128 @@
+// @flow
+// Implementation of `sourcecred export-graph`.
+
+import {Graph} from "../core/graph";
+import * as NullUtil from "../util/null";
+import * as RepoIdRegistry from "../core/repoIdRegistry";
+import {repoIdToString, stringToRepoId, type RepoId} from "../core/repoId";
+import dedent from "../util/dedent";
+import type {Command} from "./command";
+import * as Common from "./common";
+import stringify from "json-stable-stringify";
+
+import type {IAnalysisAdapter} from "../analysis/analysisAdapter";
+import {AnalysisAdapter as GithubAnalysisAdapter} from "../plugins/github/analysisAdapter";
+import {AnalysisAdapter as GitAnalysisAdapter} from "../plugins/git/analysisAdapter";
+
+function usage(print: (string) => void): void {
+  print(
+    dedent`\
+    usage: sourcecred export-graph REPO_ID [--help]
+
+    Print a combined SourceCred graph for a given REPO_ID.
+    Data must already be loaded for the given REPO_ID, using
+    'sourcecred load REPO_ID'
+
+    REPO_ID refers to a GitHub repository in the form OWNER/NAME: for
+    example, torvalds/linux. The REPO_ID may be a "combined" repo as
+    created by the --output flag to sourcecred load.
+
+    Arguments:
+        REPO_ID
+            Already-loaded repository for which to load data.
+
+        --help
+            Show this help message and exit, as 'sourcecred help export-graph'.
+
+    Environment Variables:
+        SOURCECRED_DIRECTORY
+            Directory owned by SourceCred, in which data, caches,
+            registries, etc. are stored. Optional: defaults to a
+            directory 'sourcecred' under your OS's temporary directory;
+            namely:
+                ${Common.defaultSourcecredDirectory()}
+    `.trimRight()
+  );
+}
+
+function die(std, message) {
+  std.err("fatal: " + message);
+  std.err("fatal: run 'sourcecred help export-graph' for help");
+  return 1;
+}
+
+export function makeExportGraph(
+  adapters: $ReadOnlyArray<IAnalysisAdapter>
+): Command {
+  return async function exportGraph(args, std) {
+    let repoId: RepoId | null = null;
+    if (adapters.length === 0) {
+      std.err("fatal: no plugins available");
+      std.err("fatal: this is likely a build error");
+      return 1;
+    }
+    for (let i = 0; i < args.length; i++) {
+      switch (args[i]) {
+        case "--help": {
+          usage(std.out);
+          return 0;
+        }
+        default: {
+          if (repoId != null)
+            return die(std, "multiple repository IDs provided");
+          // Should be a repository.
+          repoId = stringToRepoId(args[i]);
+          break;
+        }
+      }
+    }
+
+    if (repoId == null) {
+      return die(std, "no repository ID provided");
+    }
+
+    const directory = Common.sourcecredDirectory();
+    const registry = RepoIdRegistry.getRegistry(directory);
+    if (RepoIdRegistry.getEntry(registry, repoId) == null) {
+      const repoIdStr = repoIdToString(repoId);
+      std.err(`fatal: repository ID ${repoIdStr} not loaded`);
+      std.err(`Try running \`sourcecred load ${repoIdStr}\` first.`);
+      return 1;
+    }
+    async function graphForAdapter(adapter: IAnalysisAdapter): Promise<Graph> {
+      try {
+        return await adapter.load(directory, NullUtil.get(repoId));
+      } catch (e) {
+        throw new Error(
+          `plugin "${adapter.declaration().name}" errored: ${e.message}`
+        );
+      }
+    }
+    let graphs: Graph[];
+    try {
+      graphs = await Promise.all(adapters.map(graphForAdapter));
+    } catch (e) {
+      return die(std, e.message);
+    }
+    const graph = Graph.merge(graphs);
+    const graphJSON = graph.toJSON();
+    std.out(stringify(graphJSON));
+    return 0;
+  };
+}
+
+const defaultAdapters = [new GithubAnalysisAdapter(), new GitAnalysisAdapter()];
+
+export const exportGraph = makeExportGraph(defaultAdapters);
+
+export const help: Command = async (args, std) => {
+  if (args.length === 0) {
+    usage(std.out);
+    return 0;
+  } else {
+    usage(std.err);
+    return 1;
+  }
+};
+
+export default exportGraph;

src/cli/exportGraph.test.js

@@ -0,0 +1,203 @@
+// @flow
+
+import tmp from "tmp";
+
+import {run} from "./testUtil";
+import {help, makeExportGraph} from "./exportGraph";
+import {Graph, NodeAddress, EdgeAddress} from "../core/graph";
+import type {IAnalysisAdapter} from "../analysis/analysisAdapter";
+import stringify from "json-stable-stringify";
+
+import * as RepoIdRegistry from "../core/repoIdRegistry";
+import {makeRepoId, type RepoId} from "../core/repoId";
+
+class MockAnalysisAdapter implements IAnalysisAdapter {
+  _resolutionGraph: ?Graph;
+  _name: string;
+
+  /**
+   * Takes a name for the plugin, and a graph that
+   * is provided as a result of a successful load.
+   * If no graph is provided, then load will fail.
+   */
+  constructor(name: string, resolutionGraph: ?Graph) {
+    this._name = name;
+    this._resolutionGraph = resolutionGraph;
+  }
+
+  declaration() {
+    return {
+      name: this._name,
+      nodePrefix: NodeAddress.empty,
+      edgePrefix: EdgeAddress.empty,
+      nodeTypes: [],
+      edgeTypes: [],
+    };
+  }
+
+  async load(
+    _unused_sourcecredDirectory: string,
+    _unused_repoId: RepoId
+  ): Promise<Graph> {
+    if (this._resolutionGraph != null) {
+      return this._resolutionGraph;
+    } else {
+      throw new Error("MockAnalysisAdapterRejects");
+    }
+  }
+}
+
+describe("cli/exportGraph", () => {
+  describe("'help' command", () => {
+    it("prints usage when given no arguments", async () => {
+      expect(await run(help, [])).toEqual({
+        exitCode: 0,
+        stdout: expect.arrayContaining([
+          expect.stringMatching(/^usage: sourcecred export-graph/),
+        ]),
+        stderr: [],
+      });
+    });
+    it("fails when given arguments", async () => {
+      expect(await run(help, ["foo/bar"])).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: expect.arrayContaining([
+          expect.stringMatching(/^usage: sourcecred export-graph/),
+        ]),
+      });
+    });
+  });
+
+  describe("'exportGraph' command", () => {
+    function setUpRegistryWithId(repoId: RepoId) {
+      const dirname = tmp.dirSync().name;
+      process.env.SOURCECRED_DIRECTORY = dirname;
+      const registry = RepoIdRegistry.addEntry(RepoIdRegistry.emptyRegistry(), {
+        repoId,
+      });
+      RepoIdRegistry.writeRegistry(registry, dirname);
+      return dirname;
+    }
+
+    it("prints usage with '--help'", async () => {
+      const exportGraph = makeExportGraph([new MockAnalysisAdapter("foo")]);
+      expect(await run(exportGraph, ["--help"])).toEqual({
+        exitCode: 0,
+        stdout: expect.arrayContaining([
+          expect.stringMatching(/^usage: sourcecred export-graph/),
+        ]),
+        stderr: [],
+      });
+    });
+
+    it("errors if no repoId is provided", async () => {
+      const exportGraph = makeExportGraph([new MockAnalysisAdapter("foo")]);
+      expect(await run(exportGraph, [])).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: expect.arrayContaining([
+          "fatal: no repository ID provided",
+          "fatal: run 'sourcecred help export-graph' for help",
+        ]),
+      });
+    });
+
+    it("throws an error if no plugins are available", async () => {
+      const exportGraph = makeExportGraph([]);
+      expect(await run(exportGraph, ["--help"])).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: [
+          "fatal: no plugins available",
+          "fatal: this is likely a build error",
+        ],
+      });
+    });
+
+    it("prints json-serialized graph to stdout for a single plugin", async () => {
+      const g = new Graph().addNode(NodeAddress.empty);
+      const mockAdapter = new MockAnalysisAdapter("foo", g);
+      const exportGraph = makeExportGraph([mockAdapter]);
+      setUpRegistryWithId(makeRepoId("foo", "bar"));
+      const result = run(exportGraph, ["foo/bar"]);
+      expect(await result).toEqual({
+        exitCode: 0,
+        stdout: [stringify(g.toJSON())],
+        stderr: [],
+      });
+    });
+
+    it("merges graphs for multiple plugins", async () => {
+      const g1 = new Graph().addNode(NodeAddress.fromParts(["g1"]));
+      const g2 = new Graph().addNode(NodeAddress.fromParts(["g2"]));
+      const m1 = new MockAnalysisAdapter("foo", g1);
+      const m2 = new MockAnalysisAdapter("bar", g2);
+      const mergedGraph = Graph.merge([g1, g2]);
+      setUpRegistryWithId(makeRepoId("foo", "bar"));
+      const exportGraph = makeExportGraph([m1, m2]);
+      expect(await run(exportGraph, ["foo/bar"])).toEqual({
+        exitCode: 0,
+        stdout: [stringify(mergedGraph.toJSON())],
+        stderr: [],
+      });
+    });
+
+    it("errors if multiple repos are provided", async () => {
+      const m1 = new MockAnalysisAdapter("foo");
+      const m2 = new MockAnalysisAdapter("bar");
+      const exportGraph = makeExportGraph([m1, m2]);
+      expect(await run(exportGraph, ["foo/bar", "zod/zoink"])).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: [
+          "fatal: multiple repository IDs provided",
+          "fatal: run 'sourcecred help export-graph' for help",
+        ],
+      });
+    });
+
+    it("errors if the repoId was not loaded first", async () => {
+      const g = new Graph().addNode(NodeAddress.empty);
+      const mockAdapter = new MockAnalysisAdapter("mock", g);
+      const exportGraph = makeExportGraph([mockAdapter]);
+      setUpRegistryWithId(makeRepoId("foo", "bar"));
+      const result = run(exportGraph, ["zod/zoink"]);
+      expect(await result).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: [
+          "fatal: repository ID zod/zoink not loaded",
+          "Try running `sourcecred load zod/zoink` first.",
+        ],
+      });
+    });
+
+    it("passes the right arguments to adapter.load", async () => {
+      const mockAdapter = new MockAnalysisAdapter("zoo");
+      const exportGraph = makeExportGraph([mockAdapter]);
+      const repoId = makeRepoId("foo", "bar");
+      // $ExpectFlowError
+      mockAdapter.load = jest.fn();
+      const directory = setUpRegistryWithId(repoId);
+      await run(exportGraph, ["foo/bar"]);
+      expect(mockAdapter.load).toHaveBeenCalledWith(directory, repoId);
+    });
+
+    it("reports the failing plugin when a plugin rejects", async () => {
+      const mockAdapter = new MockAnalysisAdapter("bar");
+      const exportGraph = makeExportGraph([mockAdapter]);
+      const repoId = makeRepoId("foo", "bar");
+      setUpRegistryWithId(repoId);
+      const result = await run(exportGraph, ["foo/bar"]);
+      expect(result).toEqual({
+        exitCode: 1,
+        stdout: [],
+        stderr: [
+          'fatal: plugin "bar" errored: MockAnalysisAdapterRejects',
+          "fatal: run 'sourcecred help export-graph' for help",
+        ],
+      });
+    });
+  });
+});

src/cli/help.js

@@ -6,6 +6,7 @@ import dedent from "../util/dedent";
 
 import {help as loadHelp} from "./load";
 import {help as analyzeHelp} from "./analyze";
+import {help as exportGraphHelp} from "./exportGraph";
 
 const help: Command = async (args, std) => {
   if (args.length === 0) {
@@ -17,6 +18,7 @@ const help: Command = async (args, std) => {
     help: metaHelp,
     load: loadHelp,
     analyze: analyzeHelp,
+    "export-graph": exportGraphHelp,
   };
   if (subHelps[command] !== undefined) {
     return subHelps[command](args.slice(1), std);
@@ -27,15 +29,18 @@ const help: Command = async (args, std) => {
 };
 
 function usage(print: (string) => void): void {
+  // TODO: Make the usage function pull its list of commands
+  // from the sub-helps, to ensure that it is comprehensive
   print(
     dedent`\
     usage: sourcecred COMMAND [ARGS...]
            sourcecred [--version] [--help]
 
     Commands:
-      load      load repository data into SourceCred
+      load          load repository data into SourceCred
-      analyze   analyze cred for a loaded repository
+      analyze       analyze cred for a loaded repository
-      help      show this help message
+      export-graph  print a raw SourceCred graph
+      help          show this help message
 
     Use 'sourcecred help COMMAND' for help about an individual command.
     `.trimRight()

src/cli/help.test.js

@@ -45,6 +45,16 @@ describe("cli/help", () => {
     });
   });
 
+  it("prints help about 'sourcecred export-graph'", async () => {
+    expect(await run(help, ["export-graph"])).toEqual({
+      exitCode: 0,
+      stdout: expect.arrayContaining([
+        expect.stringMatching(/^usage: sourcecred export-graph/),
+      ]),
+      stderr: [],
+    });
+  });
+
   it("fails when given an unknown command", async () => {
     expect(await run(help, ["wat"])).toEqual({
       exitCode: 1,

src/cli/sourcecred.js

@@ -8,6 +8,7 @@ import {VERSION_SHORT} from "../core/version";
 import help from "./help";
 import load from "./load";
 import analyze from "./analyze";
+import exportGraph from "./exportGraph";
 
 const sourcecred: Command = async (args, std) => {
   if (args.length === 0) {
@@ -25,6 +26,8 @@ const sourcecred: Command = async (args, std) => {
       return load(args.slice(1), std);
     case "analyze":
       return analyze(args.slice(1), std);
+    case "export-graph":
+      return exportGraph(args.slice(1), std);
     default:
       std.err("fatal: unknown command: " + JSON.stringify(args[0]));
       std.err("fatal: run 'sourcecred help' for commands and usage");

src/cli/sourcecred.test.js

@@ -14,6 +14,7 @@ function mockCommand(name) {
 jest.mock("./help", () => mockCommand("help"));
 jest.mock("./load", () => mockCommand("load"));
 jest.mock("./analyze", () => mockCommand("analyze"));
+jest.mock("./exportGraph", () => mockCommand("export-graph"));
 
 describe("cli/sourcecred", () => {
   it("fails with usage when invoked with no arguments", async () => {
@@ -64,6 +65,16 @@ describe("cli/sourcecred", () => {
     });
   });
 
+  it("responds to 'export-graph'", async () => {
+    expect(
+      await run(sourcecred, ["export-graph", "foo/bar", "foo/baz"])
+    ).toEqual({
+      exitCode: 2,
+      stdout: ['out(export-graph): ["foo/bar","foo/baz"]'],
+      stderr: ["err(export-graph)"],
+    });
+  });
+
   it("fails given an unknown command", async () => {
     expect(await run(sourcecred, ["wat"])).toEqual({
       exitCode: 1,