From 4184e8594a80e5c1c55a66db100e3281775ac727 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 29 Jun 2018 12:12:37 -0700 Subject: [PATCH] Save the GitHub relational store from the CLI (#447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: This provides a command-line entry point `load-plugin-v3` (which will become `load-plugin` eventually), which fetches the GitHub data via GraphQL and saves the resulting `RelationalStore` to disk. A change to the Babel config is needed to prevent runtime errors of the form `_callee7` is not defined, where `_callee7` is a gensym that is appears exactly once in the source (in use position, not definition position). I’m not sure exactly what is causing the error or why this config change fixes it. But while this patch may be fragile, I don’t think that it’s likely to subtly break anything, so I’m okay with pushing it for now and dealing with any resulting breakage as it arises. Paired with @decentralion. Test Plan: Run `yarn backend`, then run something like: ``` node bin/sourcecredV3.js load-plugin-v3 \ sourcecred example-github --plugin github ``` Inspect results in `SOURCECRED_DIR/data/OWNER/NAME/github/view.json`, where `SOURCECRED_DIR` is `/tmp/sourcecred` by default, and `OWNER` and `NAME` are the repository owner and name. This example repository takes about 1.1 seconds to run. The SourceCred repository takes about 45 seconds. wchargin-branch: cli-load-plugin --- config/babel.js | 20 +++--- config/paths.js | 5 +- src/v3/cli/commands/loadPlugin.js | 87 +++++++++++++++++++++++++ src/v3/cli/common.js | 31 +++++++++ src/v3/plugins/github/loadGithubData.js | 27 ++++++++ 5 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 src/v3/cli/commands/loadPlugin.js create mode 100644 src/v3/cli/common.js create mode 100644 src/v3/plugins/github/loadGithubData.js diff --git a/config/babel.js b/config/babel.js index 4631609..266d9d4 100644 --- a/config/babel.js +++ b/config/babel.js @@ -127,15 +127,17 @@ if (env === "test") { // Must come before `babel-plugin-transform-regenerator`. require.resolve("babel-plugin-transform-es2015-for-of"), ] - : []), - // function* () { yield 42; yield 43; } - [ - require.resolve("babel-plugin-transform-regenerator"), - { - // Async functions are converted to generators by babel-preset-env - async: false, - }, - ], + : [ + // function* () { yield 42; yield 43; } + [ + require.resolve("babel-plugin-transform-regenerator"), + { + // Async functions are converted to generators by + // babel-preset-env + async: false, + }, + ], + ]), // Adds syntax support for import() require.resolve("babel-plugin-syntax-dynamic-import"), ], diff --git a/config/paths.js b/config/paths.js index 20ea194..fe9c4c8 100644 --- a/config/paths.js +++ b/config/paths.js @@ -53,13 +53,16 @@ module.exports = { // source file, and the key will be the filename of the bundled entry // point within the build directory. backendEntryPoints: { + sourcecred: resolveApp("src/v1/cli/sourcecred.js"), "commands/combine": resolveApp("src/v1/cli/commands/combine.js"), "commands/graph": resolveApp("src/v1/cli/commands/graph.js"), "commands/plugin-graph": resolveApp("src/v1/cli/commands/pluginGraph.js"), "commands/start": resolveApp("src/v1/cli/commands/start.js"), apiApp: resolveApp("src/v1/app/apiApp.js"), - sourcecred: resolveApp("src/v1/cli/sourcecred.js"), + // sourcecredV3: resolveApp("src/v3/cli/sourcecred.js"), + "commands/load-plugin-v3": resolveApp("src/v3/cli/commands/loadPlugin.js"), + // fetchAndPrintGithubRepo: resolveApp( "src/v1/plugins/github/bin/fetchAndPrintGithubRepo.js" ), diff --git a/src/v3/cli/commands/loadPlugin.js b/src/v3/cli/commands/loadPlugin.js new file mode 100644 index 0000000..7588bd8 --- /dev/null +++ b/src/v3/cli/commands/loadPlugin.js @@ -0,0 +1,87 @@ +// @flow + +import {Command, flags} from "@oclif/command"; +import mkdirp from "mkdirp"; +import path from "path"; + +import {loadGithubData} from "../../plugins/github/loadGithubData"; +import {pluginNames, sourcecredDirectoryFlag} from "../common"; + +export default class PluginGraphCommand extends Command { + static description = "load data required for a single plugin"; + + static args = [ + { + name: "repo_owner", + required: true, + description: "owner of the GitHub repository for which to fetch data", + }, + { + name: "repo_name", + required: true, + description: "name of the GitHub repository for which to fetch data", + }, + ]; + + static flags = { + plugin: flags.string({ + description: "plugin whose data to load", + required: true, + options: pluginNames(), + }), + "sourcecred-directory": sourcecredDirectoryFlag(), + "github-token": flags.string({ + description: + "a GitHub API token, as generated at " + + "https://github.com/settings/tokens/new" + + "; required only if using the GitHub plugin", + env: "SOURCECRED_GITHUB_TOKEN", + }), + }; + + async run() { + const { + args: {repo_owner: repoOwner, repo_name: repoName}, + flags: { + "github-token": githubToken, + "sourcecred-directory": basedir, + plugin, + }, + } = this.parse(PluginGraphCommand); + loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}); + } +} + +function loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}) { + const outputDirectory = path.join( + basedir, + "data", + repoOwner, + repoName, + plugin + ); + mkdirp.sync(outputDirectory); + switch (plugin) { + case "github": + if (githubToken == null) { + // TODO: This check should be abstracted so that plugins can + // specify their argument dependencies and get nicely + // formatted errors. + console.error("fatal: No GitHub token specified. Try `--help'."); + process.exitCode = 1; + return; + } else { + loadGithubData({ + token: githubToken, + repoOwner, + repoName, + outputDirectory, + }); + } + break; + default: + console.error("fatal: Unknown plugin: " + (plugin: empty)); + process.exitCode = 1; + return; + } +} diff --git a/src/v3/cli/common.js b/src/v3/cli/common.js new file mode 100644 index 0000000..eae6057 --- /dev/null +++ b/src/v3/cli/common.js @@ -0,0 +1,31 @@ +// @flow + +import {flags} from "@oclif/command"; +import os from "os"; +import path from "path"; + +export type PluginName = "github"; +export function pluginNames(): PluginName[] { + return ["github"]; +} + +function defaultStorageDirectory() { + return path.join(os.tmpdir(), "sourcecred"); +} + +export function sourcecredDirectoryFlag() { + return flags.string({ + char: "d", + description: "directory for storing graphs and other SourceCred data", + env: "SOURCECRED_DIRECTORY", + default: () => defaultStorageDirectory(), + }); +} + +export function nodeMaxOldSpaceSizeFlag() { + return flags.integer({ + description: "--max_old_space_size flag to node; increases available heap", + default: 8192, + env: "SOURCECRED_NODE_MAX_OLD_SPACE", + }); +} diff --git a/src/v3/plugins/github/loadGithubData.js b/src/v3/plugins/github/loadGithubData.js new file mode 100644 index 0000000..7b61746 --- /dev/null +++ b/src/v3/plugins/github/loadGithubData.js @@ -0,0 +1,27 @@ +// @flow + +import fs from "fs-extra"; +import path from "path"; + +import fetchGithubRepo from "./fetchGithubRepo"; +import {RelationalView} from "./relationalView"; + +export type Options = {| + +token: string, + +repoOwner: string, + +repoName: string, + +outputDirectory: string, +|}; + +export async function loadGithubData(options: Options): Promise { + const response = await fetchGithubRepo( + options.repoOwner, + options.repoName, + options.token + ); + const view = new RelationalView(); + view.addData(response); + const blob = JSON.stringify(view); + const outputFilename = path.join(options.outputDirectory, "view.json"); + return fs.writeFile(outputFilename, blob); +}