Speed up Git graph creation (#205)

Summary:
Because of `mergeConservative`’s naive implementation, using it as a
reducer results in a roughly quadratic algorithm. Replacing this with a
mutative accumulation has the same semantics but goes much faster.

Test Plan:
For correctness: tests pass. For performance: apply the following patch
to collect timing data. Then run:
```shell
$ NODE_ENV=production yarn backend
$ node bin/loadAndPrintGitRepository.js . >/tmp/sourcecred-git
$ node bin/createGitGraph.js /tmp/sourcecred-git
```
to run against the current state of SourceCred. Before this patch, the
elapsed time is about 6m00s; after this patch, it is about 0m1.3s.
Specifically:
```
$ cat timing_before
[0] Git graph creation: 239593.958ms
[1] Git graph creation: 240380.557ms
[2] Git graph creation: 241687.042ms

$ cat timing_after
[0] Git graph creation: 1585.903ms
[1] Git graph creation: 1315.430ms
[2] Git graph creation: 1373.833ms
```

<details>
<summary>Patch to collect timing data</summary>

```diff
diff --git a/config/paths.js b/config/paths.js
index f875eee..1bc1469 100644
--- a/config/paths.js
+++ b/config/paths.js
@@ -64,5 +64,6 @@ module.exports = {
     loadAndPrintGitRepository: resolveApp(
       "src/plugins/git/bin/loadAndPrintRepository.js"
     ),
+    createGitGraph: resolveApp("src/plugins/git/bin/createGraph.js"),
   },
 };
diff --git a/src/plugins/git/bin/createGraph.js b/src/plugins/git/bin/createGraph.js
new file mode 100644
index 0000000..a35ca1b
--- /dev/null
+++ b/src/plugins/git/bin/createGraph.js
@@ -0,0 +1,25 @@
+// @flow
+
+import {readFileSync} from "fs";
+
+import {createGraph} from "../createGraph";
+
+function main() {
+  const filename = process.argv[2];
+  const raw = JSON.parse(readFileSync(filename).toString());
+  const results = [];
+  for (let i = 0; i < 3; i++) {
+    const timer = `[${i}] Git graph creation`;
+    console.time(timer);
+    results.push(createGraph(raw));
+    console.timeEnd(timer);
+  }
+  console.log(
+    "Checksum: " +
+      JSON.stringify(
+        results.map((graph) => graph.nodes().length ^ graph.edges().length)
+      )
+  );
+}
+
+main();
```

</details>

wchargin-branch: collect-gold-rings
This commit is contained in:
William Chargin 2018-05-04 10:52:41 -07:00 committed by GitHub
parent 0bf4f73f86
commit d3dcf1ef5a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 1 deletions

View File

@ -50,7 +50,16 @@ class GitGraphCreator {
), ),
this.becomesEdges(repository), this.becomesEdges(repository),
]; ];
return graphs.reduce((g, h) => Graph.mergeConservative(g, h), new Graph()); const result = new Graph();
graphs.forEach((g) => {
g.nodes().forEach((node) => {
result.addNode(node);
});
g.edges().forEach((edge) => {
result.addEdge(edge);
});
});
return result;
} }
treeAndNameToSubmoduleUrls(repository: Repository) { treeAndNameToSubmoduleUrls(repository: Repository) {