Include a submodule in the main example repository (#156)

Summary:
The main example repository now covers the currently desired features:
it has blobs, subtrees, and submodules, and commits that change each of
these. (We don’t have merge commits yet—we can add those once we start
to care about them.)

Once this is merged, I will push the two repositories to GitHub.

Test Plan:
Verifying and understanding is easier than ever before. You can run the
following commands to create the repositories in question on your disk:
```shell
$ yarn backend
$ node bin/createExampleRepo.js /tmp/repo
$ node bin/createExampleRepo.js --submodule /tmp/subrepo
```

You can then explore these repositories at your leisure. For instance,
to check that the `loadRepository` snapshot has the right set of
commits, inspect the output of the following command:
```shell
$ git -C /tmp/repo log --format='%H %T'
```
Or, to check that a particular tree has the right contents, just run:
```shell
$ git -C /tmp/repo ls-tree TREE_SHA
```
Verifying the `exampleRepo` snapshot is similarly easy: just check that
the lists of commit SHAs in `/tmp/repo` and `/tmp/subrepo` are correct.

wchargin-branch: include-submodule

src/plugins/git/__snapshots__/loadRepository.test.js.snap

@@ -3,43 +3,75 @@
 exports[`loadRepository loads from HEAD 1`] = `
 Object {
   "commits": Map {
-    "677b340674bde17fdaac3b5f5eef929139ef2a52" => Object {
+    "3507b7c8690e329477c76b73f8511f78fdbafe52" => Object {
-      "hash": "677b340674bde17fdaac3b5f5eef929139ef2a52",
+      "hash": "3507b7c8690e329477c76b73f8511f78fdbafe52",
-      "treeHash": "6152a37dba8aa54dc4bc2d59c1f01c2afeba74b0",
+      "treeHash": "aeb32ba2769e46068cc09aa05723b5dd81c2e3a0",
     },
-    "4be43f1cda04e51e42fec0cfe8e1e2dff116e839" => Object {
+    "1468ff8437299dc4aa426996784be30a87226007" => Object {
-      "hash": "4be43f1cda04e51e42fec0cfe8e1e2dff116e839",
+      "hash": "1468ff8437299dc4aa426996784be30a87226007",
-      "treeHash": "93642dbd1793e84a6f529a1e1b1b4f87a4f5c878",
+      "treeHash": "77e66e35964b9c0a2cde52fb2640d3004b742feb",
     },
-    "cbb26b570d1eed3c681b8f03ff31231c1bffd6d6" => Object {
+    "d6693032379c8c55aafc72ad355db29ce71396b0" => Object {
-      "hash": "cbb26b570d1eed3c681b8f03ff31231c1bffd6d6",
+      "hash": "d6693032379c8c55aafc72ad355db29ce71396b0",
-      "treeHash": "f6736d27cd7eb7e35ae22a906854c700eb5cf6c1",
+      "treeHash": "c32b1ddc3a622e7a7e30c0b3f56a78eaed19cfab",
     },
-    "301749e9af8cd6e9aee3a49a64029b98a4695e34" => Object {
+    "fb67eef11f7f34e4ad530ce36624c47fef0cd2e2" => Object {
-      "hash": "301749e9af8cd6e9aee3a49a64029b98a4695e34",
+      "hash": "fb67eef11f7f34e4ad530ce36624c47fef0cd2e2",
-      "treeHash": "4d5f2603a4b63aa68b8e51facf542a62e4c1d065",
+      "treeHash": "b936e6133d3316250f0a91779f72c72141279941",
+    },
+    "849572148969b2c3480e8479604ccba3b189092a" => Object {
+      "hash": "849572148969b2c3480e8479604ccba3b189092a",
+      "treeHash": "e5e3b135d4f7d03e9906aead28c7757aa2b676bd",
+    },
+    "db7608c2b4eba9ca72c6e3d636a7795cf9bcb97d" => Object {
+      "hash": "db7608c2b4eba9ca72c6e3d636a7795cf9bcb97d",
+      "treeHash": "5822dfcc81bf3ef8f99f68a21a33d07386fbd7b8",
+    },
+    "486bb61573b09069dc9a55023b06e72049f8556d" => Object {
+      "hash": "486bb61573b09069dc9a55023b06e72049f8556d",
+      "treeHash": "79ee2359849733d6a51fd1b5da1cebdbaa29254f",
     },
   },
   "trees": Map {
-    "6152a37dba8aa54dc4bc2d59c1f01c2afeba74b0" => Object {
+    "aeb32ba2769e46068cc09aa05723b5dd81c2e3a0" => Object {
       "entries": Map {
+        ".gitmodules" => Object {
+          "hash": "8c6cac301e763aa6d36466e0a775eb804be2c311",
+          "name": ".gitmodules",
+          "type": "blob",
+        },
         "README.txt" => Object {
-          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
           "name": "README.txt",
           "type": "blob",
         },
+        "pygravitydefier" => Object {
+          "hash": "29ef158bc982733e2ba429fcf73e2f7562244188",
+          "name": "pygravitydefier",
+          "type": "commit",
+        },
+        "science.txt" => Object {
+          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
         "src" => Object {
           "hash": "78fc9c83023386854c6bfdc5761c0e58f68e226f",
           "name": "src",
           "type": "tree",
         },
       },
-      "hash": "6152a37dba8aa54dc4bc2d59c1f01c2afeba74b0",
+      "hash": "aeb32ba2769e46068cc09aa05723b5dd81c2e3a0",
     },
-    "93642dbd1793e84a6f529a1e1b1b4f87a4f5c878" => Object {
+    "77e66e35964b9c0a2cde52fb2640d3004b742feb" => Object {
       "entries": Map {
+        ".gitmodules" => Object {
+          "hash": "8c6cac301e763aa6d36466e0a775eb804be2c311",
+          "name": ".gitmodules",
+          "type": "blob",
+        },
         "README.txt" => Object {
-          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
           "name": "README.txt",
           "type": "blob",
         },
@@ -48,18 +80,33 @@ Object {
           "name": "TODOS.txt",
           "type": "blob",
         },
+        "pygravitydefier" => Object {
+          "hash": "29ef158bc982733e2ba429fcf73e2f7562244188",
+          "name": "pygravitydefier",
+          "type": "commit",
+        },
+        "science.txt" => Object {
+          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
         "src" => Object {
           "hash": "78fc9c83023386854c6bfdc5761c0e58f68e226f",
           "name": "src",
           "type": "tree",
         },
       },
-      "hash": "93642dbd1793e84a6f529a1e1b1b4f87a4f5c878",
+      "hash": "77e66e35964b9c0a2cde52fb2640d3004b742feb",
     },
-    "f6736d27cd7eb7e35ae22a906854c700eb5cf6c1" => Object {
+    "c32b1ddc3a622e7a7e30c0b3f56a78eaed19cfab" => Object {
       "entries": Map {
+        ".gitmodules" => Object {
+          "hash": "8c6cac301e763aa6d36466e0a775eb804be2c311",
+          "name": ".gitmodules",
+          "type": "blob",
+        },
         "README.txt" => Object {
-          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
           "name": "README.txt",
           "type": "blob",
         },
@@ -68,23 +115,108 @@ Object {
           "name": "TODOS.txt",
           "type": "blob",
         },
+        "pygravitydefier" => Object {
+          "hash": "29ef158bc982733e2ba429fcf73e2f7562244188",
+          "name": "pygravitydefier",
+          "type": "commit",
+        },
+        "science.txt" => Object {
+          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
         "src" => Object {
           "hash": "7b79d579b62994faba3b69fdf8aa442586c32681",
           "name": "src",
           "type": "tree",
         },
       },
-      "hash": "f6736d27cd7eb7e35ae22a906854c700eb5cf6c1",
+      "hash": "c32b1ddc3a622e7a7e30c0b3f56a78eaed19cfab",
     },
-    "4d5f2603a4b63aa68b8e51facf542a62e4c1d065" => Object {
+    "b936e6133d3316250f0a91779f72c72141279941" => Object {
+      "entries": Map {
+        ".gitmodules" => Object {
+          "hash": "8c6cac301e763aa6d36466e0a775eb804be2c311",
+          "name": ".gitmodules",
+          "type": "blob",
+        },
+        "README.txt" => Object {
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
+          "name": "README.txt",
+          "type": "blob",
+        },
+        "TODOS.txt" => Object {
+          "hash": "ddec7477206c30c31b81482e56b877a0b3c2638b",
+          "name": "TODOS.txt",
+          "type": "blob",
+        },
+        "pygravitydefier" => Object {
+          "hash": "762c062fbdc7ec198cd693e95d55b374a08ff3e3",
+          "name": "pygravitydefier",
+          "type": "commit",
+        },
+        "science.txt" => Object {
+          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
+        "src" => Object {
+          "hash": "7b79d579b62994faba3b69fdf8aa442586c32681",
+          "name": "src",
+          "type": "tree",
+        },
+      },
+      "hash": "b936e6133d3316250f0a91779f72c72141279941",
+    },
+    "e5e3b135d4f7d03e9906aead28c7757aa2b676bd" => Object {
+      "entries": Map {
+        ".gitmodules" => Object {
+          "hash": "8c6cac301e763aa6d36466e0a775eb804be2c311",
+          "name": ".gitmodules",
+          "type": "blob",
+        },
+        "README.txt" => Object {
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
+          "name": "README.txt",
+          "type": "blob",
+        },
+        "pygravitydefier" => Object {
+          "hash": "762c062fbdc7ec198cd693e95d55b374a08ff3e3",
+          "name": "pygravitydefier",
+          "type": "commit",
+        },
+        "science.txt" => Object {
+          "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
+      },
+      "hash": "e5e3b135d4f7d03e9906aead28c7757aa2b676bd",
+    },
+    "5822dfcc81bf3ef8f99f68a21a33d07386fbd7b8" => Object {
       "entries": Map {
         "README.txt" => Object {
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
+          "name": "README.txt",
+          "type": "blob",
+        },
+        "science.txt" => Object {
           "hash": "f1f2514ca6d7a6a1a0511957021b1995bf9ace1c",
+          "name": "science.txt",
+          "type": "blob",
+        },
+      },
+      "hash": "5822dfcc81bf3ef8f99f68a21a33d07386fbd7b8",
+    },
+    "79ee2359849733d6a51fd1b5da1cebdbaa29254f" => Object {
+      "entries": Map {
+        "README.txt" => Object {
+          "hash": "6e70bfb555b270a017c743df4cd7d35d5337e0ef",
           "name": "README.txt",
           "type": "blob",
         },
       },
-      "hash": "4d5f2603a4b63aa68b8e51facf542a62e4c1d065",
+      "hash": "79ee2359849733d6a51fd1b5da1cebdbaa29254f",
     },
     "78fc9c83023386854c6bfdc5761c0e58f68e226f" => Object {
       "entries": Map {
@@ -123,13 +255,12 @@ Object {
 exports[`loadRepository processes an old commit 1`] = `
 Object {
   "commits": Set {
-    "cbb26b570d1eed3c681b8f03ff31231c1bffd6d6",
+    "db7608c2b4eba9ca72c6e3d636a7795cf9bcb97d",
-    "301749e9af8cd6e9aee3a49a64029b98a4695e34",
+    "486bb61573b09069dc9a55023b06e72049f8556d",
   },
   "trees": Set {
-    "f6736d27cd7eb7e35ae22a906854c700eb5cf6c1",
+    "5822dfcc81bf3ef8f99f68a21a33d07386fbd7b8",
-    "4d5f2603a4b63aa68b8e51facf542a62e4c1d065",
+    "79ee2359849733d6a51fd1b5da1cebdbaa29254f",
-    "7b79d579b62994faba3b69fdf8aa442586c32681",
   },
 }
 `;

src/plugins/git/demoData/__snapshots__/exampleRepo.test.js.snap

@@ -2,10 +2,13 @@
 
 exports[`createExampleRepo is deterministic 1`] = `
 Array [
-  "301749e9af8cd6e9aee3a49a64029b98a4695e34",
+  "486bb61573b09069dc9a55023b06e72049f8556d",
-  "cbb26b570d1eed3c681b8f03ff31231c1bffd6d6",
+  "db7608c2b4eba9ca72c6e3d636a7795cf9bcb97d",
-  "4be43f1cda04e51e42fec0cfe8e1e2dff116e839",
+  "849572148969b2c3480e8479604ccba3b189092a",
-  "677b340674bde17fdaac3b5f5eef929139ef2a52",
+  "fb67eef11f7f34e4ad530ce36624c47fef0cd2e2",
+  "d6693032379c8c55aafc72ad355db29ce71396b0",
+  "1468ff8437299dc4aa426996784be30a87226007",
+  "3507b7c8690e329477c76b73f8511f78fdbafe52",
 ]
 `;
 

src/plugins/git/demoData/exampleRepo.js

@@ -1,6 +1,7 @@
 // @flow
 
 import mkdirp from "mkdirp";
+import tmp from "tmp";
 
 import {makeUtils} from "../gitUtils";
 import type {Hash} from "../types";
@@ -10,17 +11,70 @@ type RepositoryInfo = {|
   +commits: $ReadOnlyArray<Hash>, // in oldest-to-newest order
 |};
 
+// For determinism, the example repository's submodule URL must be set
+// to a fixed value. We choose the URL of the GitHub mirror (it's as
+// good as any other, and makes sense).
+export const SUBMODULE_REMOTE_URL =
+  "https://github.com/sourcecred/example-git-submodule.git";
+
+// The example repository checks out two different versions of the
+// submodule, as given by the following two hashes. These must exist
+// within the submodule; this property is checked by the test file for
+// this module.
+export const SUBMODULE_COMMIT_1: Hash =
+  "762c062fbdc7ec198cd693e95d55b374a08ff3e3";
+export const SUBMODULE_COMMIT_2: Hash =
+  "29ef158bc982733e2ba429fcf73e2f7562244188";
+
+/**
+ * Create the main example repository.
+ */
 export function createExampleRepo(intoDirectory: string): RepositoryInfo {
   const repositoryPath = intoDirectory;
   mkdirp(repositoryPath);
   const git = makeUtils(repositoryPath);
   const commits = [];
+  function commit(message) {
+    git.deterministicCommit(message);
+    commits.push(git.head());
+  }
 
   git.exec(["init"]);
 
-  git.writeAndStage("README.txt", "Amazing physics going on...\n");
+  git.writeAndStage(
-  git.deterministicCommit("Initial commit");
+    "README.txt",
-  commits.push(git.head());
+    [
+      "example-git\n",
+      "-----------\n\n",
+      "This repository provides example data for the SourceCred Git plugin.\n",
+      "Pay no attention to the contents behind the curtain.\n",
+    ].join()
+  );
+  commit("Initial commit");
+
+  git.writeAndStage("science.txt", "Amazing physics going on...\n");
+  commit("Add repository description");
+
+  const tmpdir = tmp.dirSync({unsafeCleanup: true});
+  const submoduleName = "pygravitydefier";
+  createExampleSubmoduleRepo(tmpdir.name);
+  git.exec(["submodule", "add", "--quiet", tmpdir.name, submoduleName]);
+  // After cloning the submodule, we don't need it anymore.
+  tmpdir.removeCallback();
+  // We need all our commits to be deterministic, which means that the
+  // submodule URL needs to be independent of the actual location on
+  // disk (which may be, e.g., a temporary directory).
+  git.exec([
+    "config",
+    "--file=.gitmodules",
+    `submodule.${submoduleName}.url`,
+    SUBMODULE_REMOTE_URL,
+  ]);
+  git.exec(["submodule", "sync", "--quiet"]);
+  git.exec(["add", ".gitmodules"]);
+  git.exec(["-C", submoduleName, "checkout", SUBMODULE_COMMIT_1, "--quiet"]);
+  git.exec(["add", submoduleName]);
+  commit("Add gravity defiance module");
 
   git.writeAndStage("src/index.py", "import antigravity\n");
   git.writeAndStage(
@@ -28,19 +82,20 @@ export function createExampleRepo(intoDirectory: string): RepositoryInfo {
     'raise NotImplementedError("TODO(physicists)")\n'
   );
   git.writeAndStage("TODOS.txt", "1. Resolve quantum gravity\n");
-  git.deterministicCommit("Discover gravity");
+  commit("Discover gravity");
-  commits.push(git.head());
+
+  git.exec(["-C", submoduleName, "checkout", SUBMODULE_COMMIT_2, "--quiet"]);
+  git.exec(["add", submoduleName]);
+  commit("Pull quantum gravity defiance from upstream");
 
   git.writeAndStage(
     "src/quantum_gravity.py",
     "import random\nif random.random() < 0.5:\n  import antigravity\n"
   );
-  git.deterministicCommit("Solve quantum gravity");
+  commit("Solve quantum gravity");
-  commits.push(git.head());
 
   git.exec(["rm", "TODOS.txt"]);
-  git.deterministicCommit("Clean up TODOS");
+  commit("Clean up TODOS");
-  commits.push(git.head());
 
   return {path: repositoryPath, commits};
 }
@@ -57,6 +112,10 @@ export function createExampleSubmoduleRepo(
   mkdirp(repositoryPath);
   const git = makeUtils(repositoryPath);
   const commits = [];
+  function commit(message) {
+    git.deterministicCommit(message);
+    commits.push(git.head());
+  }
 
   git.exec(["init"]);
 
@@ -69,12 +128,10 @@ export function createExampleSubmoduleRepo(
       "a submodule in a larger repository.\n",
     ].join("")
   );
-  git.deterministicCommit("Initial commit");
+  commit("Initial commit");
-  commits.push(git.head());
 
   git.writeAndStage("useless.txt", "Nothing to see here; move along.\n");
-  git.deterministicCommit("Add a file, so that we have multiple commits");
+  commit("Add a file, so that we have multiple commits");
-  commits.push(git.head());
 
   return {path: repositoryPath, commits};
 }

src/plugins/git/demoData/exampleRepo.test.js

@@ -2,7 +2,12 @@
 
 import tmp from "tmp";
 
-import {createExampleRepo, createExampleSubmoduleRepo} from "./exampleRepo";
+import {
+  createExampleRepo,
+  createExampleSubmoduleRepo,
+  SUBMODULE_COMMIT_1,
+  SUBMODULE_COMMIT_2,
+} from "./exampleRepo";
 
 const cleanups: (() => void)[] = [];
 afterAll(() => {
@@ -27,4 +32,11 @@ describe("createExampleSubmoduleRepo", () => {
   it("is deterministic", () => {
     expect(createExampleSubmoduleRepo(mkdtemp()).commits).toMatchSnapshot();
   });
+
+  it("includes all the SHAs that it should", () => {
+    const commits = createExampleSubmoduleRepo(mkdtemp()).commits;
+    expect(commits).toEqual(
+      expect.arrayContaining([SUBMODULE_COMMIT_1, SUBMODULE_COMMIT_2])
+    );
+  });
 });

src/plugins/git/loadRepository.test.js

@@ -44,34 +44,4 @@ describe("loadRepository", () => {
       trees: new Set(part.trees.keys()),
     }).toMatchSnapshot();
   });
-
-  it("works with submodules", () => {
-    const repositoryPath = mkdtemp();
-    const git = makeUtils(repositoryPath);
-
-    const subproject = createExampleRepo(mkdtemp());
-
-    git.exec(["init"]);
-    git.exec(["submodule", "--quiet", "add", subproject.path, "physics"]);
-    git.deterministicCommit("Initial commit");
-
-    const head = git.head();
-
-    const repository = loadRepository(repositoryPath, "HEAD");
-    const commit = repository.commits.get(head);
-    expect(commit).toEqual(expect.anything());
-    if (commit == null) {
-      throw new Error("Unreachable");
-    }
-    const tree = repository.trees.get(commit.treeHash);
-    expect(tree).toEqual(expect.anything());
-    if (tree == null) {
-      throw new Error("Unreachable");
-    }
-    expect(tree.entries.get("physics")).toEqual({
-      type: "commit",
-      name: "physics",
-      hash: subproject.commits[subproject.commits.length - 1],
-    });
-  });
 });