From 669f34d00996d16507a4bfdae856cf333f1f9f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Tue, 19 Mar 2019 19:00:08 -0700 Subject: [PATCH] Add `fetchGithubOrg` for loading organizations (#1117) This commit adds a module, `fetchGithubOrg`, which loads data on GitHub organizations, most notably including the list of repositories in that org. The structure of this commit is heavily influenced by review feedback from @wchargin's [review] of a related PR. Test plan: This logic depends on actually hitting GitHub's API, so the tests are modeled off the related `fetchGithubRepo` module. There is a new shell test, `src/plugins/github/fetchGithubOrgTest.sh`, which verifies that that the org loading logic works via a snapshot. To verify the correctness of this commit, I've performed the following checks: - `yarn test --full` passes - inspection of `src/plugins/github/example/example-organization.json` confirms that the list of repositories matches the repos for the "sourcecred-test-organization" organization - manually breaking the snapshot (by removing a repo from the snapshot) causes `yarn test --full` to fail - running `src/plugins/github/fetchGithubOrgTest.sh -u` restores the snapshot, afterwhich `yarn test --full` passes again. [review]: https://github.com/sourcecred/sourcecred/pull/1089#pullrequestreview-204577637 --- config/paths.js | 3 + config/test.js | 8 ++ .../github/bin/fetchAndPrintGithubOrg.js | 55 +++++++++++ .../github/example/organizations.snapshot | 28 ++++++ src/plugins/github/fetchGithubOrg.js | 71 ++++++++++++++ src/plugins/github/fetchGithubOrgTest.sh | 94 +++++++++++++++++++ 6 files changed, 259 insertions(+) create mode 100644 src/plugins/github/bin/fetchAndPrintGithubOrg.js create mode 100644 src/plugins/github/example/organizations.snapshot create mode 100644 src/plugins/github/fetchGithubOrg.js create mode 100755 src/plugins/github/fetchGithubOrgTest.sh diff --git a/config/paths.js b/config/paths.js index e1e0201..5672315 100644 --- a/config/paths.js +++ b/config/paths.js @@ -34,6 +34,9 @@ module.exports = { fetchAndPrintGithubRepo: resolveApp( "src/plugins/github/bin/fetchAndPrintGithubRepo.js" ), + fetchAndPrintGithubOrg: resolveApp( + "src/plugins/github/bin/fetchAndPrintGithubOrg.js" + ), createExampleRepo: resolveApp("src/plugins/git/bin/createExampleRepo.js"), }, }; diff --git a/config/test.js b/config/test.js index 6881bf6..3f83a65 100644 --- a/config/test.js +++ b/config/test.js @@ -112,6 +112,14 @@ function makeTasks(mode /*: "BASIC" | "FULL" */) { ]), deps: ["backend"], }, + { + id: "fetchGithubOrgTest", + cmd: withSourcecredBinEnv([ + "./src/plugins/github/fetchGithubOrgTest.sh", + "--no-build", + ]), + deps: ["backend"], + }, { id: "loadRepositoryTest", cmd: withSourcecredBinEnv([ diff --git a/src/plugins/github/bin/fetchAndPrintGithubOrg.js b/src/plugins/github/bin/fetchAndPrintGithubOrg.js new file mode 100644 index 0000000..b27d7d5 --- /dev/null +++ b/src/plugins/github/bin/fetchAndPrintGithubOrg.js @@ -0,0 +1,55 @@ +// @flow +/* + * Command-line utility to fetch GitHub data using the API in + * ../fetchGithubOrg, and print it to stdout. Useful for testing or + * saving some data to disk. + * + * Usage: + * + * node bin/fetchAndPrintGithubOrg.js ORGANIZATION_NAME GITHUB_TOKEN [PAGE_SIZE] + * + * where GITHUB_TOKEN is a GitHub authentication token, as generated + * from https://github.com/settings/tokens/new. + */ + +import stringify from "json-stable-stringify"; + +import {fetchGithubOrg} from "../fetchGithubOrg"; + +function parseArgs() { + const argv = process.argv.slice(2); + const fail = () => { + const invocation = process.argv.slice(0, 2).join(" "); + throw new Error( + `Usage: ${invocation} ORGANIZATION_NAME GITHUB_TOKEN [PAGE_SIZE]` + ); + }; + if (argv.length < 2) { + fail(); + } + const [organization, githubToken, ...rest] = argv; + let pageSize: ?number; + if (rest.length === 1) { + pageSize = Number(rest[0]); + } + const result = {organization, githubToken, pageSize}; + if (rest.length > 1) { + fail(); + } + return result; +} + +function main() { + const {organization, githubToken, pageSize} = parseArgs(); + fetchGithubOrg(organization, githubToken, pageSize) + .then((data) => { + console.log(stringify(data, {space: 4})); + }) + .catch((errors) => { + console.error("Errors processing the result:"); + console.error(errors); + process.exit(1); + }); +} + +main(); diff --git a/src/plugins/github/example/organizations.snapshot b/src/plugins/github/example/organizations.snapshot new file mode 100644 index 0000000..2b4181d --- /dev/null +++ b/src/plugins/github/example/organizations.snapshot @@ -0,0 +1,28 @@ +# results for org: sourcecred-test-organization +{ + "name": "sourcecred-test-organization", + "repos": [ + { + "name": "test-repo-two", + "owner": "sourcecred-test-organization" + }, + { + "name": "test-repo-one", + "owner": "sourcecred-test-organization" + } + ] +} + +# results for org: sourcecred-empty-organization +{ + "name": "sourcecred-empty-organization", + "repos": [ + ] +} + +# results for org: sourcecred-nonexistent-organization +{ + "name": "sourcecred-nonexistent-organization", + "repos": [ + ] +} diff --git a/src/plugins/github/fetchGithubOrg.js b/src/plugins/github/fetchGithubOrg.js new file mode 100644 index 0000000..dfa7faa --- /dev/null +++ b/src/plugins/github/fetchGithubOrg.js @@ -0,0 +1,71 @@ +// @flow + +import {type RepoId, makeRepoId} from "../../core/repoId"; +import * as Queries from "../../graphql/queries"; +import {postQuery} from "./fetchGithubRepo"; + +export type Organization = {| + +repos: $ReadOnlyArray, + +name: string, +|}; + +const DEFAULT_PAGE_SIZE = 100; + +/** + * Fetches information about a given GitHub organization. + * + * Currently just gets the ids of its repositories, but we may want additional + * information in the future. + */ +export async function fetchGithubOrg( + org: string, + token: string, + // Regular clients should leave pageSize at the default 50. + // Exposed for testing purposes. + pageSize: ?number +): Promise { + const numRepos = pageSize == null ? DEFAULT_PAGE_SIZE : pageSize; + const b = Queries.build; + const makePayload = (afterCursor: ?string) => { + const afterArg = afterCursor == null ? {} : {after: b.literal(afterCursor)}; + const args = { + query: b.variable("searchQuery"), + type: b.enumLiteral("REPOSITORY"), + first: b.literal(numRepos), + ...afterArg, + }; + return { + body: [ + b.query( + "PerformSearch", + [b.param("searchQuery", "String!")], + [ + b.field("search", args, [ + b.field("nodes", {}, [ + b.inlineFragment("Repository", [ + b.field("name"), + b.field("id"), + ]), + ]), + b.field("pageInfo", {}, [ + b.field("endCursor"), + b.field("hasNextPage"), + ]), + ]), + ] + ), + ], + variables: {searchQuery: `org:${org}`}, + }; + }; + + let result = await postQuery(makePayload(), token); + const resultNodes = [result.search.nodes]; + while (result.search.pageInfo.hasNextPage) { + const afterCursor = result.search.pageInfo.endCursor; + result = await postQuery(makePayload(afterCursor), token); + resultNodes.push(result.search.nodes); + } + const repos = [].concat(...resultNodes).map((n) => makeRepoId(org, n.name)); + return {repos, name: org}; +} diff --git a/src/plugins/github/fetchGithubOrgTest.sh b/src/plugins/github/fetchGithubOrgTest.sh new file mode 100755 index 0000000..1a61a7f --- /dev/null +++ b/src/plugins/github/fetchGithubOrgTest.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -eu + +data_file=src/plugins/github/example/organizations.snapshot + +usage() { + printf 'usage: %s [-u|--updateSnapshot] [--[no-]build] [--help]\n' "$0" + printf 'Required environment variables:\n' + printf ' GITHUB_TOKEN: A 40-character hex string API token.\n' + printf 'Flags:\n' + printf ' -u|--updateSnapshot\n' + printf ' Update the stored file instead of checking its contents\n' + printf ' --[no-]build\n' + printf ' Whether to run "yarn backend" before the test.\n' + printf ' Default is --build.\n' + printf ' --help\n' + printf ' Show this message\n' + printf '\n' + printf 'Environment variables:' + printf ' SOURCECRED_BIN\n' + printf ' When using --no-build, directory containing the SourceCred\n' + printf ' executables (output of "yarn backend"). Default is ./bin.\n' +} + +fetch() { + if [ -z "${GITHUB_TOKEN:-}" ]; then + printf >&2 'Please set the GITHUB_TOKEN environment variable\n' + printf >&2 'to a 40-character hex string API token from GitHub.\n' + return 1 + fi + # Set the PAGE_SIZE to 1 to ensure pagination logic really works. + PAGE_SIZE=1 + echo "# results for org: sourcecred-test-organization" + node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \ + sourcecred-test-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}" + echo + echo "# results for org: sourcecred-empty-organization" + node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \ + sourcecred-empty-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}" + echo + echo "# results for org: sourcecred-nonexistent-organization" + node "${SOURCECRED_BIN:-./bin}/fetchAndPrintGithubOrg.js" \ + sourcecred-nonexistent-organization "${GITHUB_TOKEN}" "${PAGE_SIZE}" +} + +check() { + output="$(mktemp)" + fetch >"${output}" + diff -uw "${data_file}" "${output}" + rm "${output}" +} + +update() { + fetch >"${data_file}" +} + +main() { + if [ -n "${SOURCECRED_BIN:-}" ]; then + BIN="$(readlink -f "${SOURCECRED_BIN}")" + fi + cd "$(git rev-parse --show-toplevel)" + UPDATE= + BUILD=1 + while [ $# -gt 0 ]; do + if [ "$1" = "--help" ]; then + usage + return 0 + elif [ "$1" = "-u" ] || [ "$1" = "--updateSnapshot" ]; then + UPDATE=1 + elif [ "$1" = "--build" ]; then + BUILD=1 + elif [ "$1" = "--no-build" ]; then + BUILD= + else + usage >&2 + return 1 + fi + shift + done + if [ -n "${BUILD}" ]; then + unset BIN + yarn backend + else + export NODE_PATH="./node_modules${NODE_PATH:+:${NODE_PATH}}" + fi + if [ -n "${UPDATE}" ]; then + update + else + check + fi +} + +main "$@"