From 2a39bd075d99a8b87cd6f06e23e92da8401701aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Thu, 13 Sep 2018 11:43:36 -0700 Subject: [PATCH] Load commit authorship from GitHub (#821) This adds logic for retrieving every commit in the default branch's history, along with authorship information connecting that commit to a GitHub user (when available). This will allows us to do better cred tracking, especially for projects that don't always use pull requests for merging code. This results in a moderate increase in load time for the GitHub plugin. On my machine, loading SourceCred before this change takes 30s, and after this change it takes 34s. Test plan: Observe that the example-github has been updated with commits and authorship. Also, I ran the query for a larger repository (`sourcecred/sourcecred`) to verify that the continuation logic works. --- .../github/__snapshots__/graphql.test.js.snap | 19 ++++ .../github/example/example-github.json | 86 +++++++++++++++++++ src/plugins/github/graphql.js | 66 +++++++++++++- 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/src/plugins/github/__snapshots__/graphql.test.js.snap b/src/plugins/github/__snapshots__/graphql.test.js.snap index 1c43f62..4b7aca0 100644 --- a/src/plugins/github/__snapshots__/graphql.test.js.snap +++ b/src/plugins/github/__snapshots__/graphql.test.js.snap @@ -216,6 +216,16 @@ exports[`plugins/github/graphql creates a query 1`] = ` pulls: pullRequests(first: 50) { ...pulls } + defaultBranchRef { + target { + __typename + ... on Commit { + history(first: 100) { + ...commitHistory + } + } + } + } } } fragment whoami on Actor { @@ -324,6 +334,15 @@ fragment reviewComments on PullRequestReviewCommentConnection { } } } +fragment commitHistory on CommitHistoryConnection { + pageInfo { + hasNextPage + endCursor + } + nodes { + ...commit + } +} fragment commit on Commit { id url diff --git a/src/plugins/github/example/example-github.json b/src/plugins/github/example/example-github.json index 23ce1dd..ea34bbf 100644 --- a/src/plugins/github/example/example-github.json +++ b/src/plugins/github/example/example-github.json @@ -1,5 +1,91 @@ { "repository": { + "defaultBranchRef": { + "target": { + "__typename": "Commit", + "history": { + "nodes": [ + { + "author": { + "user": null + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OjZiZDFiNGMwYjcxOWMyMmM2ODhhNzQ4NjNiZTA3YTY5OWI3YjliMzQ=", + "oid": "6bd1b4c0b719c22c688a74863be07a699b7b9b34", + "url": "https://github.com/sourcecred/example-github/commit/6bd1b4c0b719c22c688a74863be07a699b7b9b34" + }, + { + "author": { + "user": { + "__typename": "User", + "id": "MDQ6VXNlcjQyODE5Mzgy", + "login": "credbot", + "url": "https://github.com/credbot" + } + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OmM0MzBiZDc0NDU1MTA1Zjc3MjE1ZWNlNTE5NDUwOTRjZWVlZTZjODY=", + "oid": "c430bd74455105f77215ece51945094ceeee6c86", + "url": "https://github.com/sourcecred/example-github/commit/c430bd74455105f77215ece51945094ceeee6c86" + }, + { + "author": { + "user": { + "__typename": "User", + "id": "MDQ6VXNlcjE0MDAwMjM=", + "login": "decentralion", + "url": "https://github.com/decentralion" + } + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OjZkNWIzYWEzMWViYjY4YTA2Y2ViNDZiYmQ2Y2Y0OWI2Y2NkNmY1ZTY=", + "oid": "6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6", + "url": "https://github.com/sourcecred/example-github/commit/6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6" + }, + { + "author": { + "user": { + "__typename": "User", + "id": "MDQ6VXNlcjE0MDAwMjM=", + "login": "decentralion", + "url": "https://github.com/decentralion" + } + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OjBhMjIzMzQ2YjRlNmRlYzAxMjdiMWU2YWE4OTJjNGVlMDQyNGI2NmE=", + "oid": "0a223346b4e6dec0127b1e6aa892c4ee0424b66a", + "url": "https://github.com/sourcecred/example-github/commit/0a223346b4e6dec0127b1e6aa892c4ee0424b66a" + }, + { + "author": { + "user": { + "__typename": "User", + "id": "MDQ6VXNlcjE0MDAwMjM=", + "login": "decentralion", + "url": "https://github.com/decentralion" + } + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OmVjYzg4OWRjOTRjZjZkYTE3YWU2ZWFiNWJiN2I3MTU1ZjU3NzUxOWQ=", + "oid": "ecc889dc94cf6da17ae6eab5bb7b7155f577519d", + "url": "https://github.com/sourcecred/example-github/commit/ecc889dc94cf6da17ae6eab5bb7b7155f577519d" + }, + { + "author": { + "user": { + "__typename": "User", + "id": "MDQ6VXNlcjE0MDAwMjM=", + "login": "decentralion", + "url": "https://github.com/decentralion" + } + }, + "id": "MDY6Q29tbWl0MTIzMjU1MDA2OmVjOTFhZGI3MThhNjA0NWI0OTIzMDNmMDBkOGU4YmViOTU3ZGM3ODA=", + "oid": "ec91adb718a6045b492303f00d8e8beb957dc780", + "url": "https://github.com/sourcecred/example-github/commit/ec91adb718a6045b492303f00d8e8beb957dc780" + } + ], + "pageInfo": { + "endCursor": "6bd1b4c0b719c22c688a74863be07a699b7b9b34 5", + "hasNextPage": false + } + } + } + }, "id": "MDEwOlJlcG9zaXRvcnkxMjMyNTUwMDY=", "issues": { "nodes": [ diff --git a/src/plugins/github/graphql.js b/src/plugins/github/graphql.js index 7b4a0fb..eaa87d0 100644 --- a/src/plugins/github/graphql.js +++ b/src/plugins/github/graphql.js @@ -40,8 +40,9 @@ import type {Repo} from "../../core/repo"; * tune the page sizes of various entities to keep them comfortably * within the global capacity. * - * We use the `PAGE_LIMIT` field for the top-level page size in - * continuations. + * For the top-level page size in continuations, we use either + * `PAGE_LIMIT` or the field-specific page size (in the case of + * commit history). * * [1]: https://developer.github.com/v4/guides/resource-limitations/#node-limit */ @@ -51,6 +52,7 @@ const PAGE_SIZE_PRS = 50; const PAGE_SIZE_COMMENTS = 20; const PAGE_SIZE_REVIEWS = 10; const PAGE_SIZE_REVIEW_COMMENTS = 10; +const PAGE_SIZE_COMMIT_HISTORY = 100; /** * What's in a continuation? If we want to fetch more comments for the @@ -109,8 +111,16 @@ export type RepositoryJSON = {| +url: string, +name: string, +owner: AuthorJSON, + +defaultBranchRef: ?RefJSON, |}; +export type RefJSON = {|+target: GitObjectJSON|}; +export type GitObjectJSON = + | {|+__typename: "COMMIT", +history: ConnectionJSON|} + | {|+__typename: "TREE"|} + | {|+__typename: "BLOB"|} + | {|+__typename: "TAG"|}; + /** * The top-level GitHub query to request data about a repository. * Callers will also be interested in `createVariables`. @@ -139,6 +149,18 @@ export function createQuery(): Body { b.fragmentSpread("pulls"), ]) ), + b.field("defaultBranchRef", {}, [ + b.field("target", {}, [ + b.field("__typename"), + b.inlineFragment("Commit", [ + b.field( + "history", + {first: b.literal(PAGE_SIZE_COMMIT_HISTORY)}, + [b.fragmentSpread("commitHistory")] + ), + ]), + ]), + ]), ] ), ] @@ -249,6 +271,37 @@ function* continuationsFromRepository( destinationPath: path, }; } + if ( + result.defaultBranchRef && + result.defaultBranchRef.target.history.pageInfo.hasNextPage + ) { + yield { + enclosingNodeType: "REPOSITORY", + enclosingNodeId: nodeId, + selections: [ + b.inlineFragment("Repository", [ + b.field("defaultBranchRef", {}, [ + b.field("target", {}, [ + b.field("__typename"), + b.inlineFragment("Commit", [ + b.field( + "history", + { + first: b.literal(PAGE_SIZE_COMMIT_HISTORY), + after: b.literal( + result.defaultBranchRef.target.history.pageInfo.endCursor + ), + }, + [b.fragmentSpread("commitHistory")] + ), + ]), + ]), + ]), + ]), + ], + destinationPath: path, + }; + } if (result.issues) { for (let i = 0; i < result.issues.nodes.length; i++) { const issue = result.issues.nodes[i]; @@ -822,6 +875,14 @@ function commitFragment(): FragmentDefinition { ]); } +function commitHistoryFragment(): FragmentDefinition { + const b = build; + return b.fragment("commitHistory", "CommitHistoryConnection", [ + makePageInfo(), + b.field("nodes", {}, [b.fragmentSpread("commit")]), + ]); +} + /** * These fragments are used to construct the root query, and also to * fetch more pages of specific entity types. @@ -834,6 +895,7 @@ export function createFragments(): FragmentDefinition[] { commentsFragment(), reviewsFragment(), reviewCommentsFragment(), + commitHistoryFragment(), commitFragment(), ]; }