From 996899ade3e19f66d7189d63ecb4fe1079e17af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Fri, 1 Mar 2019 15:33:40 -0700 Subject: [PATCH] Add CLI command: `sourcecred export-graph` (#1110) * Add CLI command: `sourcecred export-graph` This adds an `export-graph` command to the SourceCred CLI. It exports the combined cred graphs for individual RepoIds, as was done for [sourcecred/research#4]. Example usage: ``` $ node bin/sourcecred.js load sourcecred/mission $ node bin/sourcecred.js export-graph sourcecred/mission > /tmp/mission_graph.json ``` Test plan: The new command is thoroughly unit tested. [sourcecred/research#4]: https://github.com/sourcecred/research/pull/4 * Address review feedback by @wchargin --- CHANGELOG.md | 1 + src/cli/exportGraph.js | 128 +++++++++++++++++++++++ src/cli/exportGraph.test.js | 203 ++++++++++++++++++++++++++++++++++++ src/cli/help.js | 11 +- src/cli/help.test.js | 10 ++ src/cli/sourcecred.js | 3 + src/cli/sourcecred.test.js | 11 ++ 7 files changed, 364 insertions(+), 3 deletions(-) create mode 100644 src/cli/exportGraph.js create mode 100644 src/cli/exportGraph.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 9beb545..8c9152b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] +- Add the `export-graph` command (#1110) - Enable loading private repositories (#1085) - Enable setting type weights to 0 in the UI (#1005) - Add support for 🚀 and 👀 reaction types (#1068) diff --git a/src/cli/exportGraph.js b/src/cli/exportGraph.js new file mode 100644 index 0000000..0e30f23 --- /dev/null +++ b/src/cli/exportGraph.js @@ -0,0 +1,128 @@ +// @flow +// Implementation of `sourcecred export-graph`. + +import {Graph} from "../core/graph"; +import * as NullUtil from "../util/null"; +import * as RepoIdRegistry from "../core/repoIdRegistry"; +import {repoIdToString, stringToRepoId, type RepoId} from "../core/repoId"; +import dedent from "../util/dedent"; +import type {Command} from "./command"; +import * as Common from "./common"; +import stringify from "json-stable-stringify"; + +import type {IAnalysisAdapter} from "../analysis/analysisAdapter"; +import {AnalysisAdapter as GithubAnalysisAdapter} from "../plugins/github/analysisAdapter"; +import {AnalysisAdapter as GitAnalysisAdapter} from "../plugins/git/analysisAdapter"; + +function usage(print: (string) => void): void { + print( + dedent`\ + usage: sourcecred export-graph REPO_ID [--help] + + Print a combined SourceCred graph for a given REPO_ID. + Data must already be loaded for the given REPO_ID, using + 'sourcecred load REPO_ID' + + REPO_ID refers to a GitHub repository in the form OWNER/NAME: for + example, torvalds/linux. The REPO_ID may be a "combined" repo as + created by the --output flag to sourcecred load. + + Arguments: + REPO_ID + Already-loaded repository for which to load data. + + --help + Show this help message and exit, as 'sourcecred help export-graph'. + + Environment Variables: + SOURCECRED_DIRECTORY + Directory owned by SourceCred, in which data, caches, + registries, etc. are stored. Optional: defaults to a + directory 'sourcecred' under your OS's temporary directory; + namely: + ${Common.defaultSourcecredDirectory()} + `.trimRight() + ); +} + +function die(std, message) { + std.err("fatal: " + message); + std.err("fatal: run 'sourcecred help export-graph' for help"); + return 1; +} + +export function makeExportGraph( + adapters: $ReadOnlyArray +): Command { + return async function exportGraph(args, std) { + let repoId: RepoId | null = null; + if (adapters.length === 0) { + std.err("fatal: no plugins available"); + std.err("fatal: this is likely a build error"); + return 1; + } + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case "--help": { + usage(std.out); + return 0; + } + default: { + if (repoId != null) + return die(std, "multiple repository IDs provided"); + // Should be a repository. + repoId = stringToRepoId(args[i]); + break; + } + } + } + + if (repoId == null) { + return die(std, "no repository ID provided"); + } + + const directory = Common.sourcecredDirectory(); + const registry = RepoIdRegistry.getRegistry(directory); + if (RepoIdRegistry.getEntry(registry, repoId) == null) { + const repoIdStr = repoIdToString(repoId); + std.err(`fatal: repository ID ${repoIdStr} not loaded`); + std.err(`Try running \`sourcecred load ${repoIdStr}\` first.`); + return 1; + } + async function graphForAdapter(adapter: IAnalysisAdapter): Promise { + try { + return await adapter.load(directory, NullUtil.get(repoId)); + } catch (e) { + throw new Error( + `plugin "${adapter.declaration().name}" errored: ${e.message}` + ); + } + } + let graphs: Graph[]; + try { + graphs = await Promise.all(adapters.map(graphForAdapter)); + } catch (e) { + return die(std, e.message); + } + const graph = Graph.merge(graphs); + const graphJSON = graph.toJSON(); + std.out(stringify(graphJSON)); + return 0; + }; +} + +const defaultAdapters = [new GithubAnalysisAdapter(), new GitAnalysisAdapter()]; + +export const exportGraph = makeExportGraph(defaultAdapters); + +export const help: Command = async (args, std) => { + if (args.length === 0) { + usage(std.out); + return 0; + } else { + usage(std.err); + return 1; + } +}; + +export default exportGraph; diff --git a/src/cli/exportGraph.test.js b/src/cli/exportGraph.test.js new file mode 100644 index 0000000..523f357 --- /dev/null +++ b/src/cli/exportGraph.test.js @@ -0,0 +1,203 @@ +// @flow + +import tmp from "tmp"; + +import {run} from "./testUtil"; +import {help, makeExportGraph} from "./exportGraph"; +import {Graph, NodeAddress, EdgeAddress} from "../core/graph"; +import type {IAnalysisAdapter} from "../analysis/analysisAdapter"; +import stringify from "json-stable-stringify"; + +import * as RepoIdRegistry from "../core/repoIdRegistry"; +import {makeRepoId, type RepoId} from "../core/repoId"; + +class MockAnalysisAdapter implements IAnalysisAdapter { + _resolutionGraph: ?Graph; + _name: string; + + /** + * Takes a name for the plugin, and a graph that + * is provided as a result of a successful load. + * If no graph is provided, then load will fail. + */ + constructor(name: string, resolutionGraph: ?Graph) { + this._name = name; + this._resolutionGraph = resolutionGraph; + } + + declaration() { + return { + name: this._name, + nodePrefix: NodeAddress.empty, + edgePrefix: EdgeAddress.empty, + nodeTypes: [], + edgeTypes: [], + }; + } + + async load( + _unused_sourcecredDirectory: string, + _unused_repoId: RepoId + ): Promise { + if (this._resolutionGraph != null) { + return this._resolutionGraph; + } else { + throw new Error("MockAnalysisAdapterRejects"); + } + } +} + +describe("cli/exportGraph", () => { + describe("'help' command", () => { + it("prints usage when given no arguments", async () => { + expect(await run(help, [])).toEqual({ + exitCode: 0, + stdout: expect.arrayContaining([ + expect.stringMatching(/^usage: sourcecred export-graph/), + ]), + stderr: [], + }); + }); + it("fails when given arguments", async () => { + expect(await run(help, ["foo/bar"])).toEqual({ + exitCode: 1, + stdout: [], + stderr: expect.arrayContaining([ + expect.stringMatching(/^usage: sourcecred export-graph/), + ]), + }); + }); + }); + + describe("'exportGraph' command", () => { + function setUpRegistryWithId(repoId: RepoId) { + const dirname = tmp.dirSync().name; + process.env.SOURCECRED_DIRECTORY = dirname; + const registry = RepoIdRegistry.addEntry(RepoIdRegistry.emptyRegistry(), { + repoId, + }); + RepoIdRegistry.writeRegistry(registry, dirname); + return dirname; + } + + it("prints usage with '--help'", async () => { + const exportGraph = makeExportGraph([new MockAnalysisAdapter("foo")]); + expect(await run(exportGraph, ["--help"])).toEqual({ + exitCode: 0, + stdout: expect.arrayContaining([ + expect.stringMatching(/^usage: sourcecred export-graph/), + ]), + stderr: [], + }); + }); + + it("errors if no repoId is provided", async () => { + const exportGraph = makeExportGraph([new MockAnalysisAdapter("foo")]); + expect(await run(exportGraph, [])).toEqual({ + exitCode: 1, + stdout: [], + stderr: expect.arrayContaining([ + "fatal: no repository ID provided", + "fatal: run 'sourcecred help export-graph' for help", + ]), + }); + }); + + it("throws an error if no plugins are available", async () => { + const exportGraph = makeExportGraph([]); + expect(await run(exportGraph, ["--help"])).toEqual({ + exitCode: 1, + stdout: [], + stderr: [ + "fatal: no plugins available", + "fatal: this is likely a build error", + ], + }); + }); + + it("prints json-serialized graph to stdout for a single plugin", async () => { + const g = new Graph().addNode(NodeAddress.empty); + const mockAdapter = new MockAnalysisAdapter("foo", g); + const exportGraph = makeExportGraph([mockAdapter]); + setUpRegistryWithId(makeRepoId("foo", "bar")); + const result = run(exportGraph, ["foo/bar"]); + expect(await result).toEqual({ + exitCode: 0, + stdout: [stringify(g.toJSON())], + stderr: [], + }); + }); + + it("merges graphs for multiple plugins", async () => { + const g1 = new Graph().addNode(NodeAddress.fromParts(["g1"])); + const g2 = new Graph().addNode(NodeAddress.fromParts(["g2"])); + const m1 = new MockAnalysisAdapter("foo", g1); + const m2 = new MockAnalysisAdapter("bar", g2); + const mergedGraph = Graph.merge([g1, g2]); + setUpRegistryWithId(makeRepoId("foo", "bar")); + const exportGraph = makeExportGraph([m1, m2]); + expect(await run(exportGraph, ["foo/bar"])).toEqual({ + exitCode: 0, + stdout: [stringify(mergedGraph.toJSON())], + stderr: [], + }); + }); + + it("errors if multiple repos are provided", async () => { + const m1 = new MockAnalysisAdapter("foo"); + const m2 = new MockAnalysisAdapter("bar"); + const exportGraph = makeExportGraph([m1, m2]); + expect(await run(exportGraph, ["foo/bar", "zod/zoink"])).toEqual({ + exitCode: 1, + stdout: [], + stderr: [ + "fatal: multiple repository IDs provided", + "fatal: run 'sourcecred help export-graph' for help", + ], + }); + }); + + it("errors if the repoId was not loaded first", async () => { + const g = new Graph().addNode(NodeAddress.empty); + const mockAdapter = new MockAnalysisAdapter("mock", g); + const exportGraph = makeExportGraph([mockAdapter]); + setUpRegistryWithId(makeRepoId("foo", "bar")); + const result = run(exportGraph, ["zod/zoink"]); + expect(await result).toEqual({ + exitCode: 1, + stdout: [], + stderr: [ + "fatal: repository ID zod/zoink not loaded", + "Try running `sourcecred load zod/zoink` first.", + ], + }); + }); + + it("passes the right arguments to adapter.load", async () => { + const mockAdapter = new MockAnalysisAdapter("zoo"); + const exportGraph = makeExportGraph([mockAdapter]); + const repoId = makeRepoId("foo", "bar"); + // $ExpectFlowError + mockAdapter.load = jest.fn(); + const directory = setUpRegistryWithId(repoId); + await run(exportGraph, ["foo/bar"]); + expect(mockAdapter.load).toHaveBeenCalledWith(directory, repoId); + }); + + it("reports the failing plugin when a plugin rejects", async () => { + const mockAdapter = new MockAnalysisAdapter("bar"); + const exportGraph = makeExportGraph([mockAdapter]); + const repoId = makeRepoId("foo", "bar"); + setUpRegistryWithId(repoId); + const result = await run(exportGraph, ["foo/bar"]); + expect(result).toEqual({ + exitCode: 1, + stdout: [], + stderr: [ + 'fatal: plugin "bar" errored: MockAnalysisAdapterRejects', + "fatal: run 'sourcecred help export-graph' for help", + ], + }); + }); + }); +}); diff --git a/src/cli/help.js b/src/cli/help.js index bae50e5..218c94c 100644 --- a/src/cli/help.js +++ b/src/cli/help.js @@ -6,6 +6,7 @@ import dedent from "../util/dedent"; import {help as loadHelp} from "./load"; import {help as analyzeHelp} from "./analyze"; +import {help as exportGraphHelp} from "./exportGraph"; const help: Command = async (args, std) => { if (args.length === 0) { @@ -17,6 +18,7 @@ const help: Command = async (args, std) => { help: metaHelp, load: loadHelp, analyze: analyzeHelp, + "export-graph": exportGraphHelp, }; if (subHelps[command] !== undefined) { return subHelps[command](args.slice(1), std); @@ -27,15 +29,18 @@ const help: Command = async (args, std) => { }; function usage(print: (string) => void): void { + // TODO: Make the usage function pull its list of commands + // from the sub-helps, to ensure that it is comprehensive print( dedent`\ usage: sourcecred COMMAND [ARGS...] sourcecred [--version] [--help] Commands: - load load repository data into SourceCred - analyze analyze cred for a loaded repository - help show this help message + load load repository data into SourceCred + analyze analyze cred for a loaded repository + export-graph print a raw SourceCred graph + help show this help message Use 'sourcecred help COMMAND' for help about an individual command. `.trimRight() diff --git a/src/cli/help.test.js b/src/cli/help.test.js index fcf5fa1..067bbac 100644 --- a/src/cli/help.test.js +++ b/src/cli/help.test.js @@ -45,6 +45,16 @@ describe("cli/help", () => { }); }); + it("prints help about 'sourcecred export-graph'", async () => { + expect(await run(help, ["export-graph"])).toEqual({ + exitCode: 0, + stdout: expect.arrayContaining([ + expect.stringMatching(/^usage: sourcecred export-graph/), + ]), + stderr: [], + }); + }); + it("fails when given an unknown command", async () => { expect(await run(help, ["wat"])).toEqual({ exitCode: 1, diff --git a/src/cli/sourcecred.js b/src/cli/sourcecred.js index 422921d..b8de7ce 100644 --- a/src/cli/sourcecred.js +++ b/src/cli/sourcecred.js @@ -8,6 +8,7 @@ import {VERSION_SHORT} from "../core/version"; import help from "./help"; import load from "./load"; import analyze from "./analyze"; +import exportGraph from "./exportGraph"; const sourcecred: Command = async (args, std) => { if (args.length === 0) { @@ -25,6 +26,8 @@ const sourcecred: Command = async (args, std) => { return load(args.slice(1), std); case "analyze": return analyze(args.slice(1), std); + case "export-graph": + return exportGraph(args.slice(1), std); default: std.err("fatal: unknown command: " + JSON.stringify(args[0])); std.err("fatal: run 'sourcecred help' for commands and usage"); diff --git a/src/cli/sourcecred.test.js b/src/cli/sourcecred.test.js index 2195d4d..51b3471 100644 --- a/src/cli/sourcecred.test.js +++ b/src/cli/sourcecred.test.js @@ -14,6 +14,7 @@ function mockCommand(name) { jest.mock("./help", () => mockCommand("help")); jest.mock("./load", () => mockCommand("load")); jest.mock("./analyze", () => mockCommand("analyze")); +jest.mock("./exportGraph", () => mockCommand("export-graph")); describe("cli/sourcecred", () => { it("fails with usage when invoked with no arguments", async () => { @@ -64,6 +65,16 @@ describe("cli/sourcecred", () => { }); }); + it("responds to 'export-graph'", async () => { + expect( + await run(sourcecred, ["export-graph", "foo/bar", "foo/baz"]) + ).toEqual({ + exitCode: 2, + stdout: ['out(export-graph): ["foo/bar","foo/baz"]'], + stderr: ["err(export-graph)"], + }); + }); + it("fails given an unknown command", async () => { expect(await run(sourcecred, ["wat"])).toEqual({ exitCode: 1,