Create a Repo type and use throughout the project (#555)

Our data model orients on getting repos from GitHub, which are
alternatively represented as strings like "sourcecred/sourcecred", or
pairs of variables representing the owner and name, or objects with
owner and name properties. We also have a few different implementations
of repo validation, which are not applied consistently.

This commit changes all that. We now have a consistent Repo type which
is an object containing a string owner and string name. Thanks to a
clever suggestion by @wchargin, it is implemented as an opaque subtype
of an object containing those properties, so that the only valid way to
construct a Repo typed object is to use one of the functions that
consistently validates the repo.

As a fly-by fix, I noticed that there were some functions in the GitHub
query generation that didn't properly mark arguments as readOnly. I've
fixed these.

Test plan: No externally-observable behavior changes (except insofar as
there is a slight change in variable names in the GitHub graphql query,
which has also resulted in a snapshot diff). `yarn travis --full`
passes. `git grep repoOwner` presents no hits.
This commit is contained in:
Dandelion Mané 2018-07-27 21:30:50 -07:00 committed by GitHub
parent dd09e28d6e
commit 4406c96c95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 201 additions and 180 deletions

View File

@ -17,7 +17,7 @@ import {type EdgeEvaluator} from "../../core/attribution/pagerank";
import {WeightConfig} from "./WeightConfig"; import {WeightConfig} from "./WeightConfig";
import type {PagerankNodeDecomposition} from "../../core/attribution/pagerankNodeDecomposition"; import type {PagerankNodeDecomposition} from "../../core/attribution/pagerankNodeDecomposition";
import RepositorySelect from "./RepositorySelect"; import RepositorySelect from "./RepositorySelect";
import type {Repo} from "./repoRegistry"; import type {Repo} from "../../core/repo";
import * as NullUtil from "../../util/null"; import * as NullUtil from "../../util/null";
@ -135,15 +135,13 @@ export class App extends React.Component<Props, State> {
} }
const githubPromise = new GithubAdapter() const githubPromise = new GithubAdapter()
.load(selectedRepo.owner, selectedRepo.name) .load(selectedRepo)
.then((adapter) => { .then((adapter) => {
const graph = adapter.graph(); const graph = adapter.graph();
return {graph, adapter}; return {graph, adapter};
}); });
const gitPromise = new GitAdapter() const gitPromise = new GitAdapter().load(selectedRepo).then((adapter) => {
.load(selectedRepo.owner, selectedRepo.name)
.then((adapter) => {
const graph = adapter.graph(); const graph = adapter.graph();
return {graph, adapter}; return {graph, adapter};
}); });

View File

@ -81,7 +81,7 @@ async function example() {
backwardName: "is fooed by", backwardName: "is fooed by",
}, },
], ],
load: (_unused_repoOwner, _unused_repoName) => { load: (_unused_repo) => {
throw new Error("unused"); throw new Error("unused");
}, },
}), }),
@ -109,7 +109,7 @@ async function example() {
backwardName: "is barred by", backwardName: "is barred by",
}, },
], ],
load: (_unused_repoOwner, _unused_repoName) => { load: (_unused_repo) => {
throw new Error("unused"); throw new Error("unused");
}, },
}), }),
@ -125,7 +125,7 @@ async function example() {
edgePrefix: () => EdgeAddress.fromParts(["xox"]), edgePrefix: () => EdgeAddress.fromParts(["xox"]),
nodeTypes: () => [], nodeTypes: () => [],
edgeTypes: () => [], edgeTypes: () => [],
load: (_unused_repoOwner, _unused_repoName) => { load: (_unused_repo) => {
throw new Error("unused"); throw new Error("unused");
}, },
}), }),
@ -141,7 +141,7 @@ async function example() {
nodeTypes: () => [], nodeTypes: () => [],
edgeTypes: () => [], edgeTypes: () => [],
name: () => "unused", name: () => "unused",
load: (_unused_repoOwner, _unused_repoName) => { load: (_unused_repo) => {
throw new Error("unused"); throw new Error("unused");
}, },
}), }),

View File

@ -7,7 +7,8 @@ import deepEqual from "lodash.isequal";
import * as NullUtil from "../../util/null"; import * as NullUtil from "../../util/null";
import type {LocalStore} from "../localStore"; import type {LocalStore} from "../localStore";
import {type Repo, fromJSON, REPO_REGISTRY_API} from "./repoRegistry"; import {fromJSON, REPO_REGISTRY_API} from "./repoRegistry";
import {type Repo, stringToRepo, repoToString} from "../../core/repo";
export const REPO_KEY = "selectedRepository"; export const REPO_KEY = "selectedRepository";
export type Status = export type Status =
@ -66,26 +67,6 @@ export default class RepositorySelect extends React.Component<Props, State> {
} }
} }
function validateRepo(repo: Repo) {
const validRe = /^[A-Za-z0-9_-]+$/;
if (!repo.owner.match(validRe)) {
throw new Error(`Invalid repository owner: ${JSON.stringify(repo.owner)}`);
}
if (!repo.name.match(validRe)) {
throw new Error(`Invalid repository name: ${JSON.stringify(repo.name)}`);
}
}
function repoStringToRepo(x: string): Repo {
const pieces = x.split("/");
if (pieces.length !== 2) {
throw new Error(`Invalid repo string: ${x}`);
}
const repo = {owner: pieces[0], name: pieces[1]};
validateRepo(repo);
return repo;
}
export async function loadStatus(localStore: LocalStore): Promise<Status> { export async function loadStatus(localStore: LocalStore): Promise<Status> {
try { try {
const response = await fetch(REPO_REGISTRY_API); const response = await fetch(REPO_REGISTRY_API);
@ -147,15 +128,14 @@ export class PureRepositorySelect extends React.PureComponent<
<span>Please choose a repository to inspect:</span>{" "} <span>Please choose a repository to inspect:</span>{" "}
{selectedRepo != null && ( {selectedRepo != null && (
<select <select
value={`${selectedRepo.owner}/${selectedRepo.name}`} value={repoToString(selectedRepo)}
onChange={(e) => { onChange={(e) => {
const repoString = e.target.value; const repo = stringToRepo(e.target.value);
const repo = repoStringToRepo(repoString);
this.props.onChange(repo); this.props.onChange(repo);
}} }}
> >
{availableRepos.map(({owner, name}) => { {availableRepos.map((repo) => {
const repoString = `${owner}/${name}`; const repoString = repoToString(repo);
return ( return (
<option value={repoString} key={repoString}> <option value={repoString} key={repoString}>
{repoString} {repoString}

View File

@ -13,6 +13,7 @@ import RepositorySelect, {
} from "./RepositorySelect"; } from "./RepositorySelect";
import {toJSON, type RepoRegistry, REPO_REGISTRY_API} from "./repoRegistry"; import {toJSON, type RepoRegistry, REPO_REGISTRY_API} from "./repoRegistry";
import {makeRepo} from "../../core/repo";
require("../testUtil").configureEnzyme(); require("../testUtil").configureEnzyme();
require("../testUtil").configureAphrodite(); require("../testUtil").configureAphrodite();
@ -63,10 +64,7 @@ describe("app/credExplorer/RepositorySelect", () => {
); );
}); });
it("renders a select with all available repos as options", () => { it("renders a select with all available repos as options", () => {
const availableRepos = [ const availableRepos = [makeRepo("foo", "bar"), makeRepo("zod", "zoink")];
{owner: "foo", name: "bar"},
{owner: "zod", name: "zoink"},
];
const selectedRepo = availableRepos[0]; const selectedRepo = availableRepos[0];
const e = shallow( const e = shallow(
<PureRepositorySelect <PureRepositorySelect
@ -78,10 +76,7 @@ describe("app/credExplorer/RepositorySelect", () => {
expect(options.map((x) => x.text())).toEqual(["foo/bar", "zod/zoink"]); expect(options.map((x) => x.text())).toEqual(["foo/bar", "zod/zoink"]);
}); });
it("the selectedRepo is selected", () => { it("the selectedRepo is selected", () => {
const availableRepos = [ const availableRepos = [makeRepo("foo", "bar"), makeRepo("zod", "zoink")];
{owner: "foo", name: "bar"},
{owner: "zod", name: "zoink"},
];
const selectedRepo = availableRepos[0]; const selectedRepo = availableRepos[0];
const e = shallow( const e = shallow(
<PureRepositorySelect <PureRepositorySelect
@ -92,10 +87,7 @@ describe("app/credExplorer/RepositorySelect", () => {
expect(e.find("select").prop("value")).toBe("foo/bar"); expect(e.find("select").prop("value")).toBe("foo/bar");
}); });
it("clicking an option triggers the onChange", () => { it("clicking an option triggers the onChange", () => {
const availableRepos = [ const availableRepos = [makeRepo("foo", "bar"), makeRepo("zod", "zoink")];
{owner: "foo", name: "bar"},
{owner: "zod", name: "zoink"},
];
const onChange = jest.fn(); const onChange = jest.fn();
const e = shallow( const e = shallow(
<PureRepositorySelect <PureRepositorySelect
@ -133,15 +125,15 @@ describe("app/credExplorer/RepositorySelect", () => {
}); });
} }
it("calls fetch and handles a simple success", () => { it("calls fetch and handles a simple success", () => {
mockRegistry([{owner: "foo", name: "bar"}]); const repo = makeRepo("foo", "bar");
const repo = {owner: "foo", name: "bar"}; mockRegistry([repo]);
return expectLoadValidStatus(testLocalStore(), [repo], repo); return expectLoadValidStatus(testLocalStore(), [repo], repo);
}); });
it("returns repos in sorted order, and selects the last repo", () => { it("returns repos in sorted order, and selects the last repo", () => {
const repos = [ const repos = [
{owner: "a", name: "b"}, makeRepo("a", "b"),
{owner: "a", name: "z"}, makeRepo("a", "z"),
{owner: "foo", name: "bar"}, makeRepo("foo", "bar"),
]; ];
const nonSortedRepos = [repos[2], repos[0], repos[1]]; const nonSortedRepos = [repos[2], repos[0], repos[1]];
mockRegistry(nonSortedRepos); mockRegistry(nonSortedRepos);
@ -176,9 +168,9 @@ describe("app/credExplorer/RepositorySelect", () => {
}); });
it("loads selectedRepo from localStore, if available", () => { it("loads selectedRepo from localStore, if available", () => {
const repos = [ const repos = [
{owner: "a", name: "b"}, makeRepo("a", "b"),
{owner: "a", name: "z"}, makeRepo("a", "z"),
{owner: "foo", name: "bar"}, makeRepo("foo", "bar"),
]; ];
mockRegistry(repos); mockRegistry(repos);
const localStore = testLocalStore(); const localStore = testLocalStore();
@ -187,9 +179,9 @@ describe("app/credExplorer/RepositorySelect", () => {
}); });
it("ignores selectedRepo from localStore, if not available", () => { it("ignores selectedRepo from localStore, if not available", () => {
const repos = [ const repos = [
{owner: "a", name: "b"}, makeRepo("a", "b"),
{owner: "a", name: "z"}, makeRepo("a", "z"),
{owner: "foo", name: "bar"}, makeRepo("foo", "bar"),
]; ];
mockRegistry(repos); mockRegistry(repos);
const localStore = testLocalStore(); const localStore = testLocalStore();
@ -198,9 +190,9 @@ describe("app/credExplorer/RepositorySelect", () => {
}); });
it("ignores malformed value in localStore", () => { it("ignores malformed value in localStore", () => {
const repos = [ const repos = [
{owner: "a", name: "b"}, makeRepo("a", "b"),
{owner: "a", name: "z"}, makeRepo("a", "z"),
{owner: "foo", name: "bar"}, makeRepo("foo", "bar"),
]; ];
mockRegistry(repos); mockRegistry(repos);
const localStore = testLocalStore(); const localStore = testLocalStore();
@ -273,7 +265,7 @@ describe("app/credExplorer/RepositorySelect", () => {
describe("RepositorySelect", () => { describe("RepositorySelect", () => {
it("initially renders a LocalStoreRepositorySelect with status LOADING", () => { it("initially renders a LocalStoreRepositorySelect with status LOADING", () => {
mockRegistry([{owner: "irrelevant", name: "unused"}]); mockRegistry([makeRepo("irrelevant", "unused")]);
const e = shallow( const e = shallow(
<RepositorySelect onChange={jest.fn()} localStore={testLocalStore()} /> <RepositorySelect onChange={jest.fn()} localStore={testLocalStore()} />
); );
@ -296,7 +288,7 @@ describe("app/credExplorer/RepositorySelect", () => {
it("on successful load, sets the status on the child", async () => { it("on successful load, sets the status on the child", async () => {
const onChange = jest.fn(); const onChange = jest.fn();
const selectedRepo = {owner: "foo", name: "bar"}; const selectedRepo = makeRepo("foo", "bar");
mockRegistry([selectedRepo]); mockRegistry([selectedRepo]);
const e = shallow( const e = shallow(
<RepositorySelect onChange={onChange} localStore={testLocalStore()} /> <RepositorySelect onChange={onChange} localStore={testLocalStore()} />
@ -313,10 +305,7 @@ describe("app/credExplorer/RepositorySelect", () => {
it("on successful load, passes the status to the onChange", async () => { it("on successful load, passes the status to the onChange", async () => {
const onChange = jest.fn(); const onChange = jest.fn();
const repo = { const repo = makeRepo("foo", "bar");
owner: "foo",
name: "bar",
};
mockRegistry([repo]); mockRegistry([repo]);
const e = shallow( const e = shallow(
<RepositorySelect onChange={onChange} localStore={testLocalStore()} /> <RepositorySelect onChange={onChange} localStore={testLocalStore()} />
@ -342,7 +331,7 @@ describe("app/credExplorer/RepositorySelect", () => {
it("child onChange triggers parent onChange", () => { it("child onChange triggers parent onChange", () => {
const onChange = jest.fn(); const onChange = jest.fn();
const repo = {owner: "foo", name: "bar"}; const repo = makeRepo("foo", "bar");
mockRegistry([repo]); mockRegistry([repo]);
const e = mount( const e = mount(
<RepositorySelect onChange={onChange} localStore={testLocalStore()} /> <RepositorySelect onChange={onChange} localStore={testLocalStore()} />
@ -355,7 +344,7 @@ describe("app/credExplorer/RepositorySelect", () => {
it("selecting child option updates top-level state", async () => { it("selecting child option updates top-level state", async () => {
const onChange = jest.fn(); const onChange = jest.fn();
const repos = [{owner: "foo", name: "bar"}, {owner: "z", name: "a"}]; const repos = [makeRepo("foo", "bar"), makeRepo("z", "a")];
mockRegistry(repos); mockRegistry(repos);
const e = mount( const e = mount(
<RepositorySelect onChange={onChange} localStore={testLocalStore()} /> <RepositorySelect onChange={onChange} localStore={testLocalStore()} />

View File

@ -5,13 +5,13 @@
// (src/app/credExplorer/RepositorySelect.js) // (src/app/credExplorer/RepositorySelect.js)
import deepEqual from "lodash.isequal"; import deepEqual from "lodash.isequal";
import {toCompat, fromCompat, type Compatible} from "../../util/compat"; import {toCompat, fromCompat, type Compatible} from "../../util/compat";
import type {Repo} from "../../core/repo";
export const REPO_REGISTRY_FILE = "repositoryRegistry.json"; export const REPO_REGISTRY_FILE = "repositoryRegistry.json";
export const REPO_REGISTRY_API = "/api/v1/data/repositoryRegistry.json"; export const REPO_REGISTRY_API = "/api/v1/data/repositoryRegistry.json";
const REPO_REGISTRY_COMPAT = {type: "REPO_REGISTRY", version: "0.1.0"}; const REPO_REGISTRY_COMPAT = {type: "REPO_REGISTRY", version: "0.1.0"};
export type Repo = {|+name: string, +owner: string|};
export type RepoRegistry = $ReadOnlyArray<Repo>; export type RepoRegistry = $ReadOnlyArray<Repo>;
export type RepoRegistryJSON = Compatible<RepoRegistry>; export type RepoRegistryJSON = Compatible<RepoRegistry>;

View File

@ -8,8 +8,9 @@ import {
type RepoRegistry, type RepoRegistry,
} from "./repoRegistry"; } from "./repoRegistry";
import {makeRepo} from "../../core/repo";
describe("app/credExplorer/repoRegistry", () => { describe("app/credExplorer/repoRegistry", () => {
const r = (owner, name) => ({owner, name});
describe("to/fromJSON compose on", () => { describe("to/fromJSON compose on", () => {
function checkExample(x: RepoRegistry) { function checkExample(x: RepoRegistry) {
expect(fromJSON(toJSON(x))).toEqual(x); expect(fromJSON(toJSON(x))).toEqual(x);
@ -19,31 +20,31 @@ describe("app/credExplorer/repoRegistry", () => {
checkExample(emptyRegistry()); checkExample(emptyRegistry());
}); });
it("nonempty registry", () => { it("nonempty registry", () => {
checkExample([r("foo", "bar"), r("zoo", "zod")]); checkExample([makeRepo("foo", "bar"), makeRepo("zoo", "zod")]);
}); });
}); });
describe("addRepo", () => { describe("addRepo", () => {
it("adds to empty registry", () => { it("adds to empty registry", () => {
expect(addRepo(r("foo", "bar"), emptyRegistry())).toEqual([ expect(addRepo(makeRepo("foo", "bar"), emptyRegistry())).toEqual([
r("foo", "bar"), makeRepo("foo", "bar"),
]); ]);
}); });
it("adds to nonempty registry", () => { it("adds to nonempty registry", () => {
const registry = [r("foo", "bar")]; const registry = [makeRepo("foo", "bar")];
expect(addRepo(r("zoo", "zod"), registry)).toEqual([ expect(addRepo(makeRepo("zoo", "zod"), registry)).toEqual([
r("foo", "bar"), makeRepo("foo", "bar"),
r("zoo", "zod"), makeRepo("zoo", "zod"),
]); ]);
}); });
it("adding repo that is already the last has no effect", () => { it("adding repo that is already the last has no effect", () => {
const registry = [r("zoo", "zod"), r("foo", "bar")]; const registry = [makeRepo("zoo", "zod"), makeRepo("foo", "bar")];
expect(addRepo(r("foo", "bar"), registry)).toEqual(registry); expect(addRepo(makeRepo("foo", "bar"), registry)).toEqual(registry);
}); });
it("adding already-existing repo shifts it to the end", () => { it("adding already-existing repo shifts it to the end", () => {
const registry = [r("zoo", "zod"), r("foo", "bar")]; const registry = [makeRepo("zoo", "zod"), makeRepo("foo", "bar")];
expect(addRepo(r("zoo", "zod"), registry)).toEqual([ expect(addRepo(makeRepo("zoo", "zod"), registry)).toEqual([
r("foo", "bar"), makeRepo("foo", "bar"),
r("zoo", "zod"), makeRepo("zoo", "zod"),
]); ]);
}); });
}); });

View File

@ -1,6 +1,7 @@
// @flow // @flow
import type {Graph, NodeAddressT, EdgeAddressT} from "../core/graph"; import type {Graph, NodeAddressT, EdgeAddressT} from "../core/graph";
import type {Repo} from "../core/repo";
export interface StaticPluginAdapter { export interface StaticPluginAdapter {
name(): string; name(): string;
@ -16,7 +17,7 @@ export interface StaticPluginAdapter {
+backwardName: string, +backwardName: string,
+prefix: EdgeAddressT, +prefix: EdgeAddressT,
|}>; |}>;
load(repoOwner: string, repoName: string): Promise<DynamicPluginAdapter>; load(repo: Repo): Promise<DynamicPluginAdapter>;
} }
export interface DynamicPluginAdapter { export interface DynamicPluginAdapter {

View File

@ -14,6 +14,8 @@ import {
sourcecredDirectoryFlag, sourcecredDirectoryFlag,
} from "../common"; } from "../common";
import {makeRepo} from "../../core/repo";
import { import {
toJSON, toJSON,
fromJSON, fromJSON,
@ -59,7 +61,7 @@ export default class PluginGraphCommand extends Command {
async run() { async run() {
const { const {
args: {repo_owner: repoOwner, repo_name: repoName}, args: {repo_owner: owner, repo_name: name},
flags: { flags: {
"github-token": githubToken, "github-token": githubToken,
"sourcecred-directory": basedir, "sourcecred-directory": basedir,
@ -67,28 +69,22 @@ export default class PluginGraphCommand extends Command {
plugin, plugin,
}, },
} = this.parse(PluginGraphCommand); } = this.parse(PluginGraphCommand);
const repo = makeRepo(owner, name);
if (!plugin) { if (!plugin) {
loadAllPlugins({ loadAllPlugins({
basedir, basedir,
plugin, plugin,
repoOwner, repo,
repoName,
githubToken, githubToken,
maxOldSpaceSize, maxOldSpaceSize,
}); });
} else { } else {
loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}); loadPlugin({basedir, plugin, repo, githubToken});
} }
} }
} }
function loadAllPlugins({ function loadAllPlugins({basedir, repo, githubToken, maxOldSpaceSize}) {
basedir,
repoOwner,
repoName,
githubToken,
maxOldSpaceSize,
}) {
if (githubToken == null) { if (githubToken == null) {
// TODO: This check should be abstracted so that plugins can // TODO: This check should be abstracted so that plugins can
// specify their argument dependencies and get nicely // specify their argument dependencies and get nicely
@ -105,8 +101,8 @@ function loadAllPlugins({
`--max_old_space_size=${maxOldSpaceSize}`, `--max_old_space_size=${maxOldSpaceSize}`,
"./bin/sourcecred.js", "./bin/sourcecred.js",
"load", "load",
repoOwner, repo.owner,
repoName, repo.name,
"--plugin", "--plugin",
pluginName, pluginName,
"--github-token", "--github-token",
@ -117,18 +113,18 @@ function loadAllPlugins({
]; ];
execDependencyGraph(tasks, {taskPassLabel: "DONE"}).then(({success}) => { execDependencyGraph(tasks, {taskPassLabel: "DONE"}).then(({success}) => {
if (success) { if (success) {
addToRepoRegistry({basedir, repoOwner, repoName}); addToRepoRegistry({basedir, repo});
} }
process.exitCode = success ? 0 : 1; process.exitCode = success ? 0 : 1;
}); });
} }
function loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}) { function loadPlugin({basedir, plugin, repo, githubToken}) {
const outputDirectory = path.join( const outputDirectory = path.join(
basedir, basedir,
"data", "data",
repoOwner, repo.owner,
repoName, repo.name,
plugin plugin
); );
mkdirp.sync(outputDirectory); mkdirp.sync(outputDirectory);
@ -144,14 +140,13 @@ function loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}) {
} else { } else {
loadGithubData({ loadGithubData({
token: githubToken, token: githubToken,
repoOwner, repo,
repoName,
outputDirectory, outputDirectory,
}); });
} }
break; break;
case "git": case "git":
loadGitData({repoOwner, repoName, outputDirectory}); loadGitData({repo, outputDirectory});
break; break;
default: default:
console.error("fatal: Unknown plugin: " + (plugin: empty)); console.error("fatal: Unknown plugin: " + (plugin: empty));
@ -163,8 +158,7 @@ function loadPlugin({basedir, plugin, repoOwner, repoName, githubToken}) {
function addToRepoRegistry(options) { function addToRepoRegistry(options) {
// TODO: Make this function transactional before loading repositories in // TODO: Make this function transactional before loading repositories in
// parallel. // parallel.
const {basedir, repoOwner, repoName} = options; const {basedir, repo} = options;
const repo = {owner: repoOwner, name: repoName};
const outputFile = path.join(basedir, REPO_REGISTRY_FILE); const outputFile = path.join(basedir, REPO_REGISTRY_FILE);
let registry = null; let registry = null;
if (fs.existsSync(outputFile)) { if (fs.existsSync(outputFile)) {

29
src/core/repo.js Normal file
View File

@ -0,0 +1,29 @@
// @flow
export opaque type Repo: {|+name: string, +owner: string|} = {|
+name: string,
+owner: string,
|};
export function makeRepo(owner: string, name: string): Repo {
const validRe = /^[A-Za-z0-9-.]+$/;
if (!owner.match(validRe)) {
throw new Error(`Invalid repository owner: ${JSON.stringify(owner)}`);
}
if (!name.match(validRe)) {
throw new Error(`Invalid repository name: ${JSON.stringify(name)}`);
}
return {owner, name};
}
export function stringToRepo(x: string): Repo {
const pieces = x.split("/");
if (pieces.length !== 2) {
throw new Error(`Invalid repo string: ${x}`);
}
return makeRepo(pieces[0], pieces[1]);
}
export function repoToString(x: Repo): string {
return `${x.owner}/${x.name}`;
}

51
src/core/repo.test.js Normal file
View File

@ -0,0 +1,51 @@
// @flow
import {makeRepo, stringToRepo, repoToString, type Repo} from "./repo";
describe("core/repo", () => {
describe("Repo type", () => {
it("manually constructing a Repo is illegal", () => {
// $ExpectFlowError
const _unused_repo: Repo = {owner: "foo", name: "bar"};
});
it("destructuring repo properties is legal", () => {
const repo: Repo = makeRepo("foo", "bar");
const _unused_owner: string = repo.owner;
const _unused_name: string = repo.name;
});
});
describe("makeRepoRepo", () => {
it("allows a simple repo", () => {
makeRepo("sourcecred", "sourcecred");
});
it("allows a repo with periods in name", () => {
makeRepo("sourcecred", "sourcecred.github.io");
});
it("allows a repo with hyphens", () => {
makeRepo("foo", "something-good");
});
it("disallows a repo with no owner", () => {
expect(() => makeRepo("", "foo")).toThrow("Invalid");
});
it("disallows a repo with no name", () => {
expect(() => makeRepo("foo", "")).toThrow("Invalid");
});
it("disallows a repo with underscores", () => {
expect(() => makeRepo("yep", "something_bad")).toThrow("Invalid");
});
});
describe("repo<->string", () => {
function testInvertible(owner, name) {
const repo = makeRepo(owner, name);
const string = `${owner}/${name}`;
expect(stringToRepo(string)).toEqual(repo);
expect(repoToString(repo)).toEqual(string);
}
it("works for simple case", () => {
testInvertible("sourcecred", "sourcecred");
});
it("works for a complicated case", () => {
testInvertible("fooolio", "foo-bar.bar-99");
});
});
});

View File

@ -413,11 +413,11 @@ Array [
"name": "FetchData", "name": "FetchData",
"params": Array [ "params": Array [
Object { Object {
"name": "repoOwner", "name": "owner",
"type": "String!", "type": "String!",
}, },
Object { Object {
"name": "repoName", "name": "name",
"type": "String!", "type": "String!",
}, },
], ],
@ -426,11 +426,11 @@ Array [
"alias": null, "alias": null,
"args": Object { "args": Object {
"name": Object { "name": Object {
"data": "repoName", "data": "name",
"type": "VARIABLE", "type": "VARIABLE",
}, },
"owner": Object { "owner": Object {
"data": "repoOwner", "data": "owner",
"type": "VARIABLE", "type": "VARIABLE",
}, },
}, },
@ -926,11 +926,11 @@ Array [
] ]
`; `;
exports[`queries end-to-end-test cases for a useful query should stringify as inline 1`] = `"query FetchData($repoOwner: String! $repoName: String!) { repository(owner: $repoOwner name: $repoName) { issues(first: 100) { pageInfo { hasNextPage } nodes { id title body number author { ...whoami } comments(first: 20) { pageInfo { hasNextPage } nodes { id author { ...whoami } body url } } } } pullRequests(first: 100) { pageInfo { hasNextPage } nodes { id title body number author { ...whoami } comments(first: 20) { pageInfo { hasNextPage } nodes { id author { ...whoami } body url } } reviews(first: 10) { pageInfo { hasNextPage } nodes { id body author { ...whoami } state comments(first: 10) { pageInfo { hasNextPage } nodes { id body author { ...whoami } } } } } } } } } fragment whoami on Actor { __typename login ... on User { id } ... on Organization { id } ... on Bot { id } }"`; exports[`queries end-to-end-test cases for a useful query should stringify as inline 1`] = `"query FetchData($owner: String! $name: String!) { repository(owner: $owner name: $name) { issues(first: 100) { pageInfo { hasNextPage } nodes { id title body number author { ...whoami } comments(first: 20) { pageInfo { hasNextPage } nodes { id author { ...whoami } body url } } } } pullRequests(first: 100) { pageInfo { hasNextPage } nodes { id title body number author { ...whoami } comments(first: 20) { pageInfo { hasNextPage } nodes { id author { ...whoami } body url } } reviews(first: 10) { pageInfo { hasNextPage } nodes { id body author { ...whoami } state comments(first: 10) { pageInfo { hasNextPage } nodes { id body author { ...whoami } } } } } } } } } fragment whoami on Actor { __typename login ... on User { id } ... on Organization { id } ... on Bot { id } }"`;
exports[`queries end-to-end-test cases for a useful query should stringify as multiline 1`] = ` exports[`queries end-to-end-test cases for a useful query should stringify as multiline 1`] = `
"query FetchData($repoOwner: String! $repoName: String!) { "query FetchData($owner: String! $name: String!) {
repository(owner: $repoOwner name: $repoName) { repository(owner: $owner name: $name) {
issues(first: 100) { issues(first: 100) {
pageInfo { pageInfo {
hasNextPage hasNextPage

View File

@ -126,11 +126,11 @@ function usefulQuery(): Body {
const body: Body = [ const body: Body = [
b.query( b.query(
"FetchData", "FetchData",
[b.param("repoOwner", "String!"), b.param("repoName", "String!")], [b.param("owner", "String!"), b.param("name", "String!")],
[ [
b.field( b.field(
"repository", "repository",
{owner: b.variable("repoOwner"), name: b.variable("repoName")}, {owner: b.variable("owner"), name: b.variable("name")},
[ [
b.field("issues", {first: b.literal(100)}, [ b.field("issues", {first: b.literal(100)}, [
makePageInfo(), makePageInfo(),

View File

@ -4,22 +4,18 @@ import tmp from "tmp";
import {localGit} from "./gitUtils"; import {localGit} from "./gitUtils";
import type {Repository} from "./types"; import type {Repository} from "./types";
import {loadRepository} from "./loadRepository"; import {loadRepository} from "./loadRepository";
import type {Repo} from "../../core/repo";
/** /**
* Load Git Repository data from a fresh clone of a GitHub repo. * Load Git Repository data from a fresh clone of a GitHub repo.
* *
* @param {String} repoOwner * @param {Repo} repo
* the GitHub username of the owner of the repository to be cloned * the GitHub repository to be cloned
* @param {String} repoName
* the name of the repository to be cloned
* @return {Repository} * @return {Repository}
* the parsed Repository from the cloned repo * the parsed Repository from the cloned repo
*/ */
export default function cloneAndLoadRepository( export default function cloneAndLoadRepository(repo: Repo): Repository {
repoOwner: string, const cloneUrl = `https://github.com/${repo.owner}/${repo.name}.git`;
repoName: string
): Repository {
const cloneUrl = `https://github.com/${repoOwner}/${repoName}.git`;
const tmpdir = tmp.dirSync({unsafeCleanup: true}); const tmpdir = tmp.dirSync({unsafeCleanup: true});
const git = localGit(tmpdir.name); const git = localGit(tmpdir.name);
git(["clone", cloneUrl, ".", "--quiet"]); git(["clone", cloneUrl, ".", "--quiet"]);

View File

@ -5,18 +5,15 @@ import path from "path";
import cloneAndLoadRepository from "./cloneAndLoadRepository"; import cloneAndLoadRepository from "./cloneAndLoadRepository";
import {createGraph} from "./createGraph"; import {createGraph} from "./createGraph";
import type {Repo} from "../../core/repo";
export type Options = {| export type Options = {|
+repoOwner: string, +repo: Repo,
+repoName: string,
+outputDirectory: string, +outputDirectory: string,
|}; |};
export function loadGitData(options: Options): Promise<void> { export function loadGitData(options: Options): Promise<void> {
const repository = cloneAndLoadRepository( const repository = cloneAndLoadRepository(options.repo);
options.repoOwner,
options.repoName
);
const graph = createGraph(repository); const graph = createGraph(repository);
const blob = JSON.stringify(graph); const blob = JSON.stringify(graph);
const outputFilename = path.join(options.outputDirectory, "graph.json"); const outputFilename = path.join(options.outputDirectory, "graph.json");

View File

@ -7,6 +7,7 @@ import {Graph} from "../../core/graph";
import * as N from "./nodes"; import * as N from "./nodes";
import * as E from "./edges"; import * as E from "./edges";
import {description} from "./render"; import {description} from "./render";
import type {Repo} from "../../core/repo";
export class StaticPluginAdapter implements IStaticPluginAdapter { export class StaticPluginAdapter implements IStaticPluginAdapter {
name() { name() {
@ -55,11 +56,8 @@ export class StaticPluginAdapter implements IStaticPluginAdapter {
}, },
]; ];
} }
async load( async load(repo: Repo): Promise<IDynamicPluginAdapter> {
repoOwner: string, const url = `/api/v1/data/data/${repo.owner}/${repo.name}/git/graph.json`;
repoName: string
): Promise<IDynamicPluginAdapter> {
const url = `/api/v1/data/data/${repoOwner}/${repoName}/git/graph.json`;
const response = await fetch(url); const response = await fetch(url);
if (!response.ok) { if (!response.ok) {
return Promise.reject(response); return Promise.reject(response);

View File

@ -202,8 +202,8 @@ Object {
`; `;
exports[`graphql creates a query 1`] = ` exports[`graphql creates a query 1`] = `
"query FetchData($repoOwner: String! $repoName: String!) { "query FetchData($owner: String! $name: String!) {
repository(owner: $repoOwner name: $repoName) { repository(owner: $owner name: $name) {
url url
name name
owner { owner {

View File

@ -14,6 +14,7 @@
import fetchGithubRepo from "../fetchGithubRepo"; import fetchGithubRepo from "../fetchGithubRepo";
import stringify from "json-stable-stringify"; import stringify from "json-stable-stringify";
import {makeRepo} from "../../../core/repo";
function parseArgs() { function parseArgs() {
const argv = process.argv.slice(2); const argv = process.argv.slice(2);
@ -24,8 +25,8 @@ function parseArgs() {
if (argv.length < 2) { if (argv.length < 2) {
fail(); fail();
} }
const [repoOwner, repoName, githubToken, ...rest] = argv; const [owner, name, githubToken, ...rest] = argv;
const result = {repoOwner, repoName, githubToken}; const result = {owner, name, githubToken};
if (rest.length > 0) { if (rest.length > 0) {
fail(); fail();
} }
@ -34,7 +35,8 @@ function parseArgs() {
function main() { function main() {
const args = parseArgs(); const args = parseArgs();
fetchGithubRepo(args.repoOwner, args.repoName, args.githubToken) const repo = makeRepo(args.owner, args.name);
fetchGithubRepo(repo, args.githubToken)
.then((data) => { .then((data) => {
console.log(stringify(data, {space: 4})); console.log(stringify(data, {space: 4}));
}) })

View File

@ -9,14 +9,13 @@ import fetch from "isomorphic-fetch";
import {stringify, inlineLayout} from "../../graphql/queries"; import {stringify, inlineLayout} from "../../graphql/queries";
import {createQuery, createVariables, postQueryExhaustive} from "./graphql"; import {createQuery, createVariables, postQueryExhaustive} from "./graphql";
import type {GithubResponseJSON} from "./graphql"; import type {GithubResponseJSON} from "./graphql";
import type {Repo} from "../../core/repo";
/** /**
* Scrape data from a GitHub repo using the GitHub API. * Scrape data from a GitHub repo using the GitHub API.
* *
* @param {String} repoOwner * @param {Repo} repo
* the GitHub username of the owner of the repository to be scraped * the GitHub repository to be scraped
* @param {String} repoName
* the name of the repository to be scraped
* @param {String} token * @param {String} token
* authentication token to be used for the GitHub API; generate a * authentication token to be used for the GitHub API; generate a
* token at: https://github.com/settings/tokens * token at: https://github.com/settings/tokens
@ -26,28 +25,18 @@ import type {GithubResponseJSON} from "./graphql";
* later * later
*/ */
export default function fetchGithubRepo( export default function fetchGithubRepo(
repoOwner: string, repo: Repo,
repoName: string,
token: string token: string
): Promise<GithubResponseJSON> { ): Promise<GithubResponseJSON> {
repoOwner = String(repoOwner);
repoName = String(repoName);
token = String(token); token = String(token);
const validName = /^[A-Za-z0-9_-]*$/;
if (!validName.test(repoOwner)) {
throw new Error(`Invalid repoOwner: ${repoOwner}`);
}
if (!validName.test(repoName)) {
throw new Error(`Invalid repoName: ${repoName}`);
}
const validToken = /^[A-Fa-f0-9]{40}$/; const validToken = /^[A-Fa-f0-9]{40}$/;
if (!validToken.test(token)) { if (!validToken.test(token)) {
throw new Error(`Invalid token: ${token}`); throw new Error(`Invalid token: ${token}`);
} }
const body = createQuery(); const body = createQuery();
const variables = createVariables(repoOwner, repoName); const variables = createVariables(repo);
const payload = {body, variables}; const payload = {body, variables};
return postQueryExhaustive( return postQueryExhaustive(
(somePayload) => postQuery(somePayload, token), (somePayload) => postQuery(somePayload, token),

View File

@ -7,6 +7,7 @@ import type {
QueryDefinition, QueryDefinition,
} from "../../graphql/queries"; } from "../../graphql/queries";
import {build} from "../../graphql/queries"; import {build} from "../../graphql/queries";
import type {Repo} from "../../core/repo";
/** /**
* This module defines the GraphQL query that we use to access the * This module defines the GraphQL query that we use to access the
@ -119,11 +120,11 @@ export function createQuery(): Body {
const body: Body = [ const body: Body = [
b.query( b.query(
"FetchData", "FetchData",
[b.param("repoOwner", "String!"), b.param("repoName", "String!")], [b.param("owner", "String!"), b.param("name", "String!")],
[ [
b.field( b.field(
"repository", "repository",
{owner: b.variable("repoOwner"), name: b.variable("repoName")}, {owner: b.variable("owner"), name: b.variable("name")},
[ [
b.field("url"), b.field("url"),
b.field("name"), b.field("name"),
@ -378,8 +379,8 @@ function* continuationsFromReview(
* results. The `postQuery` function may be called multiple times. * results. The `postQuery` function may be called multiple times.
*/ */
export async function postQueryExhaustive( export async function postQueryExhaustive(
postQuery: ({body: Body, variables: {[string]: any}}) => Promise<any>, postQuery: ({body: Body, variables: {+[string]: any}}) => Promise<any>,
payload: {body: Body, variables: {[string]: any}} payload: {body: Body, variables: {+[string]: any}}
) { ) {
const originalResult = await postQuery(payload); const originalResult = await postQuery(payload);
return resolveContinuations( return resolveContinuations(
@ -394,7 +395,7 @@ export async function postQueryExhaustive(
* resolve the continuations and return the merged results. * resolve the continuations and return the merged results.
*/ */
async function resolveContinuations( async function resolveContinuations(
postQuery: ({body: Body, variables: {[string]: any}}) => Promise<any>, postQuery: ({body: Body, variables: {+[string]: any}}) => Promise<any>,
originalResult: any, originalResult: any,
continuations: $ReadOnlyArray<Continuation> continuations: $ReadOnlyArray<Continuation>
): Promise<any> { ): Promise<any> {
@ -820,6 +821,6 @@ export function createFragments(): FragmentDefinition[] {
]; ];
} }
export function createVariables(repoOwner: string, repoName: string) { export function createVariables(repo: Repo): {+[string]: any} {
return {repoOwner, repoName}; return repo;
} }

View File

@ -13,6 +13,7 @@ import {
postQueryExhaustive, postQueryExhaustive,
requiredFragments, requiredFragments,
} from "./graphql"; } from "./graphql";
import {makeRepo} from "../../core/repo";
describe("graphql", () => { describe("graphql", () => {
describe("creates continuations", () => { describe("creates continuations", () => {
@ -944,7 +945,7 @@ describe("graphql", () => {
const result = await postQueryExhaustive(postQuery, { const result = await postQueryExhaustive(postQuery, {
body: createQuery(), body: createQuery(),
variables: createVariables("sourcecred", "discussion"), variables: createVariables(makeRepo("sourcecred", "discussion")),
}); });
expect(postQuery).toHaveBeenCalledTimes(3); expect(postQuery).toHaveBeenCalledTimes(3);

View File

@ -5,20 +5,16 @@ import path from "path";
import fetchGithubRepo from "./fetchGithubRepo"; import fetchGithubRepo from "./fetchGithubRepo";
import {RelationalView} from "./relationalView"; import {RelationalView} from "./relationalView";
import type {Repo} from "../../core/repo";
export type Options = {| export type Options = {|
+token: string, +token: string,
+repoOwner: string, +repo: Repo,
+repoName: string,
+outputDirectory: string, +outputDirectory: string,
|}; |};
export async function loadGithubData(options: Options): Promise<void> { export async function loadGithubData(options: Options): Promise<void> {
const response = await fetchGithubRepo( const response = await fetchGithubRepo(options.repo, options.token);
options.repoOwner,
options.repoName,
options.token
);
const view = new RelationalView(); const view = new RelationalView();
view.addData(response); view.addData(response);
const blob = JSON.stringify(view); const blob = JSON.stringify(view);

View File

@ -9,6 +9,7 @@ import * as N from "./nodes";
import * as E from "./edges"; import * as E from "./edges";
import {RelationalView} from "./relationalView"; import {RelationalView} from "./relationalView";
import {description} from "./render"; import {description} from "./render";
import type {Repo} from "../../core/repo";
export class StaticPluginAdapter implements IStaticPluginAdapter { export class StaticPluginAdapter implements IStaticPluginAdapter {
name() { name() {
@ -54,11 +55,8 @@ export class StaticPluginAdapter implements IStaticPluginAdapter {
}, },
]; ];
} }
async load( async load(repo: Repo): Promise<IDynamicPluginAdapater> {
repoOwner: string, const url = `/api/v1/data/data/${repo.owner}/${repo.name}/github/view.json`;
repoName: string
): Promise<IDynamicPluginAdapater> {
const url = `/api/v1/data/data/${repoOwner}/${repoName}/github/view.json`;
const response = await fetch(url); const response = await fetch(url);
if (!response.ok) { if (!response.ok) {
return Promise.reject(response); return Promise.reject(response);