Save the GitHub relational store from the CLI (#447)

Summary:
This provides a command-line entry point `load-plugin-v3` (which will
become `load-plugin` eventually), which fetches the GitHub data via
GraphQL and saves the resulting `RelationalStore` to disk.

A change to the Babel config is needed to prevent runtime errors of the
form `_callee7` is not defined, where `_callee7` is a gensym that is
appears exactly once in the source (in use position, not definition
position). I’m not sure exactly what is causing the error or why this
config change fixes it. But while this patch may be fragile, I don’t
think that it’s likely to subtly break anything, so I’m okay with
pushing it for now and dealing with any resulting breakage as it arises.

Paired with @decentralion.

Test Plan:
Run `yarn backend`, then run something like:

```
node bin/sourcecredV3.js load-plugin-v3 \
    sourcecred example-github --plugin github
```

Inspect results in `SOURCECRED_DIR/data/OWNER/NAME/github/view.json`,
where `SOURCECRED_DIR` is `/tmp/sourcecred` by default, and `OWNER` and
`NAME` are the repository owner and name.

This example repository takes about 1.1 seconds to run. The SourceCred
repository takes about 45 seconds.

wchargin-branch: cli-load-plugin

config/babel.js

@@ -127,15 +127,17 @@ if (env === "test") {
             // Must come before `babel-plugin-transform-regenerator`.
             require.resolve("babel-plugin-transform-es2015-for-of"),
           ]
-        : []),
+        : [
-      // function* () { yield 42; yield 43; }
+            // function* () { yield 42; yield 43; }
-      [
+            [
-        require.resolve("babel-plugin-transform-regenerator"),
+              require.resolve("babel-plugin-transform-regenerator"),
-        {
+              {
-          // Async functions are converted to generators by babel-preset-env
+                // Async functions are converted to generators by
-          async: false,
+                // babel-preset-env
-        },
+                async: false,
-      ],
+              },
+            ],
+          ]),
       // Adds syntax support for import()
       require.resolve("babel-plugin-syntax-dynamic-import"),
     ],

config/paths.js

@@ -53,13 +53,16 @@ module.exports = {
   // source file, and the key will be the filename of the bundled entry
   // point within the build directory.
   backendEntryPoints: {
+    sourcecred: resolveApp("src/v1/cli/sourcecred.js"),
     "commands/combine": resolveApp("src/v1/cli/commands/combine.js"),
     "commands/graph": resolveApp("src/v1/cli/commands/graph.js"),
     "commands/plugin-graph": resolveApp("src/v1/cli/commands/pluginGraph.js"),
     "commands/start": resolveApp("src/v1/cli/commands/start.js"),
     apiApp: resolveApp("src/v1/app/apiApp.js"),
-    sourcecred: resolveApp("src/v1/cli/sourcecred.js"),
+    //
     sourcecredV3: resolveApp("src/v3/cli/sourcecred.js"),
+    "commands/load-plugin-v3": resolveApp("src/v3/cli/commands/loadPlugin.js"),
+    //
     fetchAndPrintGithubRepo: resolveApp(
       "src/v1/plugins/github/bin/fetchAndPrintGithubRepo.js"
     ),

src/v3/cli/commands/loadPlugin.js

@@ -0,0 +1,87 @@
+// @flow
+
+import {Command, flags} from "@oclif/command";
+import mkdirp from "mkdirp";
+import path from "path";
+
+import {loadGithubData} from "../../plugins/github/loadGithubData";
+import {pluginNames, sourcecredDirectoryFlag} from "../common";
+
+export default class PluginGraphCommand extends Command {
+  static description = "load data required for a single plugin";
+
+  static args = [
+    {
+      name: "repo_owner",
+      required: true,
+      description: "owner of the GitHub repository for which to fetch data",
+    },
+    {
+      name: "repo_name",
+      required: true,
+      description: "name of the GitHub repository for which to fetch data",
+    },
+  ];
+
+  static flags = {
+    plugin: flags.string({
+      description: "plugin whose data to load",
+      required: true,
+      options: pluginNames(),
+    }),
+    "sourcecred-directory": sourcecredDirectoryFlag(),
+    "github-token": flags.string({
+      description:
+        "a GitHub API token, as generated at " +
+        "https://github.com/settings/tokens/new" +
+        "; required only if using the GitHub plugin",
+      env: "SOURCECRED_GITHUB_TOKEN",
+    }),
+  };
+
+  async run() {
+    const {
+      args: {repo_owner: repoOwner, repo_name: repoName},
+      flags: {
+        "github-token": githubToken,
+        "sourcecred-directory": basedir,
+        plugin,
+      },
+    } = this.parse(PluginGraphCommand);
+    loadPlugin({basedir, plugin, repoOwner, repoName, githubToken});
+  }
+}
+
+function loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}) {
+  const outputDirectory = path.join(
+    basedir,
+    "data",
+    repoOwner,
+    repoName,
+    plugin
+  );
+  mkdirp.sync(outputDirectory);
+  switch (plugin) {
+    case "github":
+      if (githubToken == null) {
+        // TODO: This check should be abstracted so that plugins can
+        // specify their argument dependencies and get nicely
+        // formatted errors.
+        console.error("fatal: No GitHub token specified. Try `--help'.");
+        process.exitCode = 1;
+        return;
+      } else {
+        loadGithubData({
+          token: githubToken,
+          repoOwner,
+          repoName,
+          outputDirectory,
+        });
+      }
+      break;
+    default:
+      console.error("fatal: Unknown plugin: " + (plugin: empty));
+      process.exitCode = 1;
+      return;
+  }
+}

src/v3/cli/common.js

@@ -0,0 +1,31 @@
+// @flow
+
+import {flags} from "@oclif/command";
+import os from "os";
+import path from "path";
+
+export type PluginName = "github";
+export function pluginNames(): PluginName[] {
+  return ["github"];
+}
+
+function defaultStorageDirectory() {
+  return path.join(os.tmpdir(), "sourcecred");
+}
+
+export function sourcecredDirectoryFlag() {
+  return flags.string({
+    char: "d",
+    description: "directory for storing graphs and other SourceCred data",
+    env: "SOURCECRED_DIRECTORY",
+    default: () => defaultStorageDirectory(),
+  });
+}
+
+export function nodeMaxOldSpaceSizeFlag() {
+  return flags.integer({
+    description: "--max_old_space_size flag to node; increases available heap",
+    default: 8192,
+    env: "SOURCECRED_NODE_MAX_OLD_SPACE",
+  });
+}

src/v3/plugins/github/loadGithubData.js

@@ -0,0 +1,27 @@
+// @flow
+
+import fs from "fs-extra";
+import path from "path";
+
+import fetchGithubRepo from "./fetchGithubRepo";
+import {RelationalView} from "./relationalView";
+
+export type Options = {|
+  +token: string,
+  +repoOwner: string,
+  +repoName: string,
+  +outputDirectory: string,
+|};
+
+export async function loadGithubData(options: Options): Promise<void> {
+  const response = await fetchGithubRepo(
+    options.repoOwner,
+    options.repoName,
+    options.token
+  );
+  const view = new RelationalView();
+  view.addData(response);
+  const blob = JSON.stringify(view);
+  const outputFilename = path.join(options.outputDirectory, "view.json");
+  return fs.writeFile(outputFilename, blob);
+}