From 2aeeca9a1391f4d7426e066aa39e31b06f64a85e Mon Sep 17 00:00:00 2001 From: William Chargin Date: Mon, 7 May 2018 12:23:09 -0700 Subject: [PATCH] Implement a command-line interface (#217) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: This commit implements the `sourcecred` command-line utility, which has three subcommands: - `plugin-graph` creates one plugin’s graph; - `combine` combines multiple on-disk graphs; and - `graph` creates all plugins’ graphs and combines them. As an implementation detail, the `into.sh` script is very convenient, avoiding needing to do any pipe management in Node (which is Not Fun). When we build for release, we may want to factor that differently. Test Plan: To see it all in action, run `yarn backend`, and then try: ``` $ export SOURCECRED_GITHUB_TOKEN="your_token_here" $ node ./bin/sourcecred.js graph sourcecred sourcecred Using output directory: /tmp/sourcecred/sourcecred Starting tasks GO create-git GO create-github PASS create-github PASS create-git GO combine PASS combine Full results PASS create-git PASS create-github PASS combine Overview Final result: SUCCESS $ ls /tmp/sourcecred/sourcecred/ graph-github.json graph-git.json graph.json $ jq '.nodes | length' /tmp/sourcecred/sourcecred/*.json 1000 7302 8302 ``` The `node sourcecred.js graph` command takes 9.8s for me. (The salient point of the last command is that the two small graphs have node count adding up to the node count of the big graph. Incidentally, we are [almost][1] at a nice round number of nodes in the GitHub graph.) [1]: https://xkcd.com/1000/ wchargin-branch: cli --- config/paths.js | 5 +- src/cli/commands/combine.js | 41 ++++++++++++ src/cli/commands/example.js | 14 ---- src/cli/commands/goodbye.js | 14 ---- src/cli/commands/graph.js | 111 ++++++++++++++++++++++++++++++++ src/cli/commands/pluginGraph.js | 87 +++++++++++++++++++++++++ src/cli/common.js | 6 ++ src/cli/into.sh | 10 +++ 8 files changed, 258 insertions(+), 30 deletions(-) create mode 100644 src/cli/commands/combine.js delete mode 100644 src/cli/commands/example.js delete mode 100644 src/cli/commands/goodbye.js create mode 100644 src/cli/commands/graph.js create mode 100644 src/cli/commands/pluginGraph.js create mode 100644 src/cli/common.js create mode 100755 src/cli/into.sh diff --git a/config/paths.js b/config/paths.js index bb8a9f9..3f4ed8b 100644 --- a/config/paths.js +++ b/config/paths.js @@ -57,8 +57,9 @@ module.exports = { // source file, and the key will be the filename of the bundled entry // point within the build directory. backendEntryPoints: { - "commands/goodbye": resolveApp("src/cli/commands/goodbye.js"), - "commands/example": resolveApp("src/cli/commands/example.js"), + "commands/combine": resolveApp("src/cli/commands/combine.js"), + "commands/graph": resolveApp("src/cli/commands/graph.js"), + "commands/plugin-graph": resolveApp("src/cli/commands/pluginGraph.js"), sourcecred: resolveApp("src/cli/sourcecred.js"), fetchAndPrintGithubRepo: resolveApp( "src/plugins/github/bin/fetchAndPrintGithubRepo.js" diff --git a/src/cli/commands/combine.js b/src/cli/commands/combine.js new file mode 100644 index 0000000..5d498b3 --- /dev/null +++ b/src/cli/commands/combine.js @@ -0,0 +1,41 @@ +// @flow + +import {Command} from "@oclif/command"; +import fs from "fs"; +import stringify from "json-stable-stringify"; +import {promisify} from "util"; + +import {Graph} from "../../core/graph"; + +export default class CombineCommand extends Command { + static description = "combine multiple contribution graphs into one big graph"; + + // We disable strict-mode so that we can accept a variable number of + // arguments. Although this is what the docs [1] suggest, it doesn't + // seem like the best way to do it. TODO: Investigate. + // + // [1]: https://oclif.io/docs/commands.html + static strict = false; + + static usage = "" + + "combine [GRAPH]...\n" + + " where each GRAPH is a JSON file generated by plugin-graph"; + + async run() { + const {argv} = this.parse(CombineCommand); + combine(argv); + } +} + +async function combine(filenames: $ReadOnlyArray) { + const readFile = promisify(fs.readFile); + const graphs = await Promise.all( + filenames.map((filename) => + readFile(filename).then((contents) => + Graph.fromJSON(JSON.parse(contents)) + ) + ) + ); + const result = Graph.mergeManyConservative(graphs); + console.log(stringify(result, {space: 4})); +} diff --git a/src/cli/commands/example.js b/src/cli/commands/example.js deleted file mode 100644 index 6a72c12..0000000 --- a/src/cli/commands/example.js +++ /dev/null @@ -1,14 +0,0 @@ -// @flow -import {Command, flags} from "@oclif/command"; - -export default class ExampleCommand extends Command { - static description = "An example command description"; - static flags = { - name: flags.string({char: "n", description: "name to print"}), - }; - async run() { - const {flags} = this.parse(ExampleCommand); - const name = flags.name || "world"; - this.log(`hello ${name} EXAMPEL`); - } -} diff --git a/src/cli/commands/goodbye.js b/src/cli/commands/goodbye.js deleted file mode 100644 index 87da6e0..0000000 --- a/src/cli/commands/goodbye.js +++ /dev/null @@ -1,14 +0,0 @@ -// @flow -import {Command, flags} from "@oclif/command"; - -export default class GoodbyeCommand extends Command { - static description = "Another example command description"; - static flags = { - name: flags.string({char: "n", description: "name to print"}), - }; - async run() { - const {flags} = this.parse(GoodbyeCommand); - const name = flags.name || "world"; - this.log(`goodbye ${name}`); - } -} diff --git a/src/cli/commands/graph.js b/src/cli/commands/graph.js new file mode 100644 index 0000000..424b257 --- /dev/null +++ b/src/cli/commands/graph.js @@ -0,0 +1,111 @@ +// @flow + +import {Command, flags} from "@oclif/command"; +import mkdirp from "mkdirp"; +import os from "os"; +import path from "path"; + +import {pluginNames} from "../common"; + +const execDependencyGraph = require("../../tools/execDependencyGraph").default; + +export default class GraphCommand extends Command { + static description = `\ +create the contribution graph for a repository + +Create the contribution graph for a repository. This creates a +contribution graph for each individual plugin, and then combines the +individual graphs into one larger graph. The graphs are stored as JSON +files under OUTPUT_DIR/REPO_OWNER/REPO_NAME, where OUTPUT_DIR is +configurable. +`.trim(); + + static args = [ + { + name: "repo_owner", + required: true, + description: "owner of the GitHub repository for which to fetch data", + }, + { + name: "repo_name", + required: true, + description: "name of the GitHub repository for which to fetch data", + }, + ]; + + static flags = { + "output-directory": flags.string({ + short: "o", + description: "directory into which to store graphs", + env: "SOURCECRED_OUTPUT_DIRECTORY", + default: path.join(os.tmpdir(), "sourcecred"), + }), + "github-token": flags.string({ + description: + "a GitHub API token, as generated at " + + "https://github.com/settings/tokens/new", + required: true, + env: "SOURCECRED_GITHUB_TOKEN", + }), + }; + + async run() { + const { + args: {repo_owner: repoOwner, repo_name: repoName}, + flags: {"github-token": token, "output-directory": outputDirectory}, + } = this.parse(GraphCommand); + graph(outputDirectory, repoOwner, repoName, token); + } +} + +function graph( + outputDirectory: string, + repoOwner: string, + repoName: string, + token: string +) { + const scopedDirectory = path.join(outputDirectory, repoOwner, repoName); + console.log("Storing graphs into: " + scopedDirectory); + mkdirp.sync(scopedDirectory); + const tasks = makeTasks(scopedDirectory, {repoOwner, repoName, token}); + execDependencyGraph(tasks).then(({success}) => { + process.exitCode = success ? 0 : 1; + }); +} + +function makeTasks(outputDirectory, {repoOwner, repoName, token}) { + const taskId = (id) => `create-${id}`; + const graphFilename = (id) => path.join(outputDirectory, `graph-${id}.json`); + const into = "./src/cli/into.sh"; + return [ + ...pluginNames().map((id) => ({ + id: taskId(id), + cmd: [ + into, + graphFilename(id), + "node", + "./bin/sourcecred.js", + "plugin-graph", + "--plugin", + id, + repoOwner, + repoName, + "--github-token", + token, + ], + deps: [], + })), + { + id: "combine", + cmd: [ + into, + path.join(outputDirectory, "graph.json"), + "node", + "./bin/sourcecred.js", + "combine", + ...pluginNames().map((id) => graphFilename(id)), + ], + deps: pluginNames().map((id) => taskId(id)), + }, + ]; +} diff --git a/src/cli/commands/pluginGraph.js b/src/cli/commands/pluginGraph.js new file mode 100644 index 0000000..2643d27 --- /dev/null +++ b/src/cli/commands/pluginGraph.js @@ -0,0 +1,87 @@ +// @flow + +import {Command, flags} from "@oclif/command"; +import stringify from "json-stable-stringify"; + +import type {Graph} from "../../core/graph"; +import type {PluginName} from "../common"; +import createGitGraph from "../../plugins/git/cloneGitGraph"; +import createGithubGraph from "../../plugins/github/fetchGithubGraph"; +import {pluginNames} from "../common"; + +export default class PluginGraphCommand extends Command { + static description = "create the contribution graph for a single plugin"; + + static args = [ + { + name: "repo_owner", + required: true, + description: "owner of the GitHub repository for which to fetch data", + }, + { + name: "repo_name", + required: true, + description: "name of the GitHub repository for which to fetch data", + }, + ]; + + static flags = { + plugin: flags.string({ + description: "plugin whose graph to generate", + required: true, + options: pluginNames(), + }), + "github-token": flags.string({ + description: + "a GitHub API token, as generated at " + + "https://github.com/settings/tokens/new" + + "; required only if using the GitHub plugin", + env: "SOURCECRED_GITHUB_TOKEN", + }), + }; + + async run() { + const { + args: {repo_owner: repoOwner, repo_name: repoName}, + flags: {"github-token": token, plugin}, + } = this.parse(PluginGraphCommand); + pluginGraph(plugin, repoOwner, repoName, token); + } +} + +function pluginGraph( + plugin: PluginName, + repoOwner: string, + repoName: string, + githubToken?: string +) { + switch (plugin) { + case "git": + display(Promise.resolve(createGitGraph(repoOwner, repoName))); + break; + case "github": + if (githubToken == null) { + // TODO: This check should be abstracted so that plugins can + // specify their argument dependencies and get nicely + // formatted errors. + console.error("fatal: No GitHub token specified. Try `--help'."); + process.exitCode = 1; + return; + } else { + display(createGithubGraph(repoOwner, repoName, githubToken)); + } + break; + default: + // eslint-disable-next-line no-unused-expressions + (plugin: empty); + console.error("fatal: Unknown plugin: " + plugin); + process.exitCode = 1; + return; + } +} + +function display(promise: Promise>) { + promise.then((graph) => { + console.log(stringify(graph, {space: 4})); + }); +} diff --git a/src/cli/common.js b/src/cli/common.js new file mode 100644 index 0000000..3312bb8 --- /dev/null +++ b/src/cli/common.js @@ -0,0 +1,6 @@ +// @flow + +export type PluginName = "git" | "github"; +export function pluginNames() { + return ["git", "github"]; +} diff --git a/src/cli/into.sh b/src/cli/into.sh new file mode 100755 index 0000000..9909f07 --- /dev/null +++ b/src/cli/into.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Utility for redirection. +set -eu +if [ $# -eq 0 ]; then + printf >&2 'into: fatal: no target provided\n' + exit 1 +fi +target="$1" +shift +exec "$@" >"${target}"