Add `fetchGithubOrg` for loading organizations (#1117)

This commit adds a module, `fetchGithubOrg`, which loads data on GitHub
organizations, most notably including the list of repositories in that
org.

The structure of this commit is heavily influenced by review feedback
from @wchargin's [review] of a related PR.

Test plan: This logic depends on actually hitting GitHub's API, so the
tests are modeled off the related `fetchGithubRepo` module. There is a
new shell test, `src/plugins/github/fetchGithubOrgTest.sh`, which
verifies that that the org loading logic works via a snapshot.

To verify the correctness of this commit, I've performed the following
checks:

- `yarn test --full` passes
- inspection of `src/plugins/github/example/example-organization.json`
confirms that the list of repositories matches the repos for the
"sourcecred-test-organization" organization
- manually breaking the snapshot (by removing a repo from the snapshot)
causes `yarn test --full` to fail
- running `src/plugins/github/fetchGithubOrgTest.sh -u` restores the
snapshot, afterwhich `yarn test --full` passes again.

[review]: https://github.com/sourcecred/sourcecred/pull/1089#pullrequestreview-204577637

config/paths.js

@@ -34,6 +34,9 @@ module.exports = {
     fetchAndPrintGithubRepo: resolveApp(
       "src/plugins/github/bin/fetchAndPrintGithubRepo.js"
     ),
+    fetchAndPrintGithubOrg: resolveApp(
+      "src/plugins/github/bin/fetchAndPrintGithubOrg.js"
+    ),
     createExampleRepo: resolveApp("src/plugins/git/bin/createExampleRepo.js"),
   },
 };

config/test.js

@@ -112,6 +112,14 @@ function makeTasks(mode /*: "BASIC" | "FULL" */) {
       ]),
       deps: ["backend"],
     },
+    {
+      id: "fetchGithubOrgTest",
+      cmd: withSourcecredBinEnv([
+        "./src/plugins/github/fetchGithubOrgTest.sh",
+        "--no-build",
+      ]),
+      deps: ["backend"],
+    },
     {
       id: "loadRepositoryTest",
       cmd: withSourcecredBinEnv([

src/plugins/github/bin/fetchAndPrintGithubOrg.js

@@ -0,0 +1,55 @@
+// @flow
+/*
+ * Command-line utility to fetch GitHub data using the API in
+ * ../fetchGithubOrg, and print it to stdout. Useful for testing or
+ * saving some data to disk.
+ *
+ * Usage:
+ *
+ *   node bin/fetchAndPrintGithubOrg.js ORGANIZATION_NAME GITHUB_TOKEN [PAGE_SIZE]
+ *
+ * where GITHUB_TOKEN is a GitHub authentication token, as generated
+ * from https://github.com/settings/tokens/new.
+ */
+
+import stringify from "json-stable-stringify";
+
+import {fetchGithubOrg} from "../fetchGithubOrg";
+
+function parseArgs() {
+  const argv = process.argv.slice(2);
+  const fail = () => {
+    const invocation = process.argv.slice(0, 2).join(" ");
+    throw new Error(
+      `Usage: ${invocation} ORGANIZATION_NAME GITHUB_TOKEN [PAGE_SIZE]`
+    );
+  };
+  if (argv.length < 2) {
+    fail();
+  }
+  const [organization, githubToken, ...rest] = argv;
+  let pageSize: ?number;
+  if (rest.length === 1) {
+    pageSize = Number(rest[0]);
+  }
+  const result = {organization, githubToken, pageSize};
+  if (rest.length > 1) {
+    fail();
+  }
+  return result;
+}
+
+function main() {
+  const {organization, githubToken, pageSize} = parseArgs();
+  fetchGithubOrg(organization, githubToken, pageSize)
+    .then((data) => {
+      console.log(stringify(data, {space: 4}));
+    })
+    .catch((errors) => {
+      console.error("Errors processing the result:");
+      console.error(errors);
+      process.exit(1);
+    });
+}
+
+main();

src/plugins/github/example/organizations.snapshot

@@ -0,0 +1,28 @@
+# results for org: sourcecred-test-organization
+{
+    "name": "sourcecred-test-organization",
+    "repos": [
+        {
+            "name": "test-repo-two",
+            "owner": "sourcecred-test-organization"
+        },
+        {
+            "name": "test-repo-one",
+            "owner": "sourcecred-test-organization"
+        }
+    ]
+}
+
+# results for org: sourcecred-empty-organization
+{
+    "name": "sourcecred-empty-organization",
+    "repos": [
+    ]
+}
+
+# results for org: sourcecred-nonexistent-organization
+{
+    "name": "sourcecred-nonexistent-organization",
+    "repos": [
+    ]
+}

src/plugins/github/fetchGithubOrg.js

@@ -0,0 +1,71 @@
+// @flow
+
+import {type RepoId, makeRepoId} from "../../core/repoId";
+import * as Queries from "../../graphql/queries";
+import {postQuery} from "./fetchGithubRepo";
+
+export type Organization = {|
+  +repos: $ReadOnlyArray<RepoId>,
+  +name: string,
+|};
+
+const DEFAULT_PAGE_SIZE = 100;
+
+/**
+ * Fetches information about a given GitHub organization.
+ *
+ * Currently just gets the ids of its repositories, but we may want additional
+ * information in the future.
+ */
+export async function fetchGithubOrg(
+  org: string,
+  token: string,
+  // Regular clients should leave pageSize at the default 50.
+  // Exposed for testing purposes.
+  pageSize: ?number
+): Promise<Organization> {
+  const numRepos = pageSize == null ? DEFAULT_PAGE_SIZE : pageSize;
+  const b = Queries.build;
+  const makePayload = (afterCursor: ?string) => {
+    const afterArg = afterCursor == null ? {} : {after: b.literal(afterCursor)};
+    const args = {
+      query: b.variable("searchQuery"),
+      type: b.enumLiteral("REPOSITORY"),
+      first: b.literal(numRepos),
+      ...afterArg,
+    };
+    return {
+      body: [
+        b.query(
+          "PerformSearch",
+          [b.param("searchQuery", "String!")],
+          [
+            b.field("search", args, [
+              b.field("nodes", {}, [
+                b.inlineFragment("Repository", [
+                  b.field("name"),
+                  b.field("id"),
+                ]),
+              ]),
+              b.field("pageInfo", {}, [
+                b.field("endCursor"),
+                b.field("hasNextPage"),
+              ]),
+            ]),
+          ]
+        ),
+      ],
+      variables: {searchQuery: `org:${org}`},
+    };
+  };
+
+  let result = await postQuery(makePayload(), token);
+  const resultNodes = [result.search.nodes];
+  while (result.search.pageInfo.hasNextPage) {
+    const afterCursor = result.search.pageInfo.endCursor;
+    result = await postQuery(makePayload(afterCursor), token);
+    resultNodes.push(result.search.nodes);
+  }
+  const repos = [].concat(...resultNodes).map((n) => makeRepoId(org, n.name));
+  return {repos, name: org};
+}

src/plugins/github/fetchGithubOrgTest.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+set -eu
+
+data_file=src/plugins/github/example/organizations.snapshot
+
+usage() {
+  printf 'usage: %s [-u|--updateSnapshot] [--[no-]build] [--help]\n' "$0"
+  printf 'Required environment variables:\n'
+  printf '  GITHUB_TOKEN: A 40-character hex string API token.\n'
+  printf 'Flags:\n'
+  printf '  -u|--updateSnapshot\n'
+  printf '      Update the stored file instead of checking its contents\n'
+  printf ' --[no-]build\n'
+  printf '      Whether to run "yarn backend" before the test.\n'
+  printf '      Default is --build.\n'
+  printf '  --help\n'
+  printf '      Show this message\n'
+  printf '\n'
+  printf 'Environment variables:'
+  printf '  SOURCECRED_BIN\n'
+  printf '      When using --no-build, directory containing the SourceCred\n'
+  printf '      executables (output of "yarn backend"). Default is ./bin.\n'
+}
+
+fetch() {
+  if [ -z "${GITHUB_TOKEN:-}" ]; then
+    printf >&2 'Please set the GITHUB_TOKEN environment variable\n'
+    printf >&2 'to a 40-character hex string API token from GitHub.\n'
+    return 1
+  fi
+  # Set the PAGE_SIZE to 1 to ensure pagination logic really works.
+  PAGE_SIZE=1
+  echo "# results for org: sourcecred-test-organization"
+  node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \
+    sourcecred-test-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}"
+  echo
+  echo "# results for org: sourcecred-empty-organization"
+  node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \
+    sourcecred-empty-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}"
+  echo
+  echo "# results for org: sourcecred-nonexistent-organization"
+  node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \
+    sourcecred-nonexistent-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}"
+}
+
+check() {
+  output="$(mktemp)"
+  fetch >"${output}"
+  diff -uw "${data_file}" "${output}"
+  rm "${output}"
+}
+
+update() {
+  fetch >"${data_file}"
+}
+
+main() {
+  if [ -n "${SOURCECRED_BIN:-}" ]; then
+    BIN="$(readlink -f "${SOURCECRED_BIN}")"
+  fi
+  cd "$(git rev-parse --show-toplevel)"
+  UPDATE=
+  BUILD=1
+  while [ $# -gt 0 ]; do
+    if [ "$1" = "--help" ]; then
+      usage
+      return 0
+    elif [ "$1" = "-u" ] || [ "$1" = "--updateSnapshot" ]; then
+      UPDATE=1
+    elif [ "$1" = "--build" ]; then
+      BUILD=1
+    elif [ "$1" = "--no-build" ]; then
+      BUILD=
+    else
+      usage >&2
+      return 1
+    fi
+    shift
+  done
+  if [ -n "${BUILD}" ]; then
+    unset BIN
+    yarn backend
+  else
+    export NODE_PATH="./node_modules${NODE_PATH:+:${NODE_PATH}}"
+  fi
+  if [ -n "${UPDATE}" ]; then
+    update
+  else
+    check
+  fi
+}
+
+main "$@"