webpack: expose repo registry at build time (#981)

Summary:
We want to remove the repository selector dropdown on the cred explorer
homepage and instead render a separate web page for each project. To do
this, we need to know which pages to render statically. We choose to
ingest this information from the state of the repository registry at
build time.

This commit adds an environment variable `REPO_REGISTRY` whose contents
are the stringified version of the repository registry, or `null` if
SourceCred has been built for the backend. This variable is defined with
Webpack’s `DefinePlugin`, so any code bundled by Webpack can refer to it
via `process.env.REPO_REGISTRY` both on the server and in the browser.

Paired with @wchargin.

Test Plan:
Sharness tests modified; running `yarn test --full` suffices.

config/env.js

@@ -7,6 +7,7 @@ const path = require("path");
 const paths = require("./paths");
 
 /*:: import type {GitState} from "../src/core/version"; */
+/*:: import type {RepoIdRegistry} from "../src/explorer/repoIdRegistry"; */
 
 // Make sure that including paths.js after env.js will read .env variables.
 delete require.cache[require.resolve("./paths")];
@@ -135,15 +136,19 @@ const SOURCECRED_FEEDBACK_URL =
     : "https://discuss.sourcecred.io/c/cred-feedback/";
 process.env.SOURCECRED_FEEDBACK_URL = SOURCECRED_FEEDBACK_URL;
 
-function getClientEnvironment() {
+function getClientEnvironment(
+  repoRegistryContents /*: RepoIdRegistry | null */
+) {
   const raw = {};
   // Useful for determining whether we’re running in production mode.
   // Most importantly, it switches React into the correct mode.
   raw.NODE_ENV = process.env.NODE_ENV || "development";
-  // Used by `src/app/version.js`.
+  // Used by `src/core/version.js`.
   raw.SOURCECRED_GIT_STATE = SOURCECRED_GIT_STATE;
-  // Used by `src/app/credExplorer/App.js`.
+  // Used by `src/explorer/App.js`.
   raw.SOURCECRED_FEEDBACK_URL = SOURCECRED_FEEDBACK_URL;
+  // Optional. Used by `src/homepage/routeData.js`
+  raw.REPO_REGISTRY = stringify(repoRegistryContents);
 
   // Stringify all values so we can feed into Webpack's DefinePlugin.
   const stringified = {"process.env": {}};

config/webpack.config.backend.js

@@ -12,7 +12,7 @@ const paths = require("./paths");
 const nodeExternals = require("webpack-node-externals");
 const getClientEnvironment = require("./env");
 
-const env = getClientEnvironment();
+const env = getClientEnvironment(null);
 
 // This is the backend configuration. It builds applications that target
 // Node and will not run in a browser.

config/webpack.config.web.js

@@ -5,7 +5,9 @@ import type {
   $Application as ExpressApp,
   $Response as ExpressResponse,
 } from "express";
+import type {RepoIdRegistry} from "../src/explorer/repoIdRegistry";
 */
+const fs = require("fs");
 const os = require("os");
 const path = require("path");
 const webpack = require("webpack");
@@ -19,8 +21,39 @@ const getClientEnvironment = require("./env");
 
 // Source maps are resource heavy and can cause out of memory issue for large source files.
 const shouldUseSourceMap = process.env.GENERATE_SOURCEMAP !== "false";
+
+function loadRepoRegistry() /*: RepoIdRegistry */ {
+  const env = process.env.SOURCECRED_DIRECTORY;
+  // TODO(#945): de-duplicate finding the directory with src/cli/common.js
+  const defaultDirectory = path.join(os.tmpdir(), "sourcecred");
+  const scDirectory = env != null ? env : defaultDirectory;
+  // TODO(@dandelion): Remove hacks around compat usage here
+  // TODO(@dandelion): Import rather than hardcode the registry file name
+  const registryFile = path.join(scDirectory, "repositoryRegistry.json");
+
+  let jsonString;
+  try {
+    jsonString = fs.readFileSync(registryFile).toString();
+  } catch (e) {
+    if (e.code === "ENOENT") {
+      jsonString = JSON.stringify([
+        {version: "0.1.0", type: "REPO_ID_REGISTRY"},
+        [],
+      ]);
+    } else {
+      throw e;
+    }
+  }
+  const json = JSON.parse(jsonString);
+  const compat = json[0];
+  if (compat.version !== "0.1.0" || compat.type !== "REPO_ID_REGISTRY") {
+    throw new Error("Compat mismatch");
+  }
+  return json[1];
+}
+
 // Get environment variables to inject into our app.
-const env = getClientEnvironment();
+const env = getClientEnvironment(loadRepoRegistry());
 
 function makeConfig(mode /*: "production" | "development" */) {
   return {

scripts/build_static_site.sh

@@ -131,7 +131,6 @@ build() {
         yarn
         yarn -s backend --output-path "${SOURCECRED_BIN}"
     fi
-    yarn -s build --output-path "${target}"
 
     if [ "${#repos[@]}" -ne 0 ]; then
         for repo in "${repos[@]}"; do
@@ -141,6 +140,8 @@ build() {
         done
     fi
 
+    yarn -s build --output-path "${target}"
+
     # Copy the SourceCred data into the appropriate API route. Using
     # `mkdir` here will fail in the case where an `api/` folder exists,
     # which is the correct behavior. (In this case, our site's

sharness/test_build_static_site.t

@@ -149,7 +149,7 @@ run_build() {
             printf >&2 "fatal: potentially unsafe argument: %s\n" "${arg}" &&
             false
         fi &&
-        run '"${flags}"' 2>err &&
+        run '"${flags}"' >out 2>err &&
         test_must_fail grep -vF \
             -e "Removing contents of build directory: " \
             -e "info: loading repository" \
@@ -230,6 +230,12 @@ test_expect_success TWO_REPOS \
     printf "2\n" | test_cmp - actual_count
 '
 
+test_expect_success TWO_REPOS \
+    "TWO_REPOS: should have a repo registry loaded into env" '
+    grep -F "REPO_REGISTRY" out &&
+    grep -xF "REPO_REGISTRY: [{\"name\":\"example-git\",\"owner\":\"sourcecred\"},{\"name\":\"example-github\",\"owner\":\"sourcecred\"}]" out
+'
+
 test_expect_success TWO_REPOS \
     "TWO_REPOS: should have data for the two repositories" '
     for repo in sourcecred/example-git sourcecred/example-github; do
@@ -265,6 +271,12 @@ test_expect_success NO_REPOS \
     test_must_fail test -e "${registry_file}"
 '
 
+test_expect_success NO_REPOS \
+    "NO_REPOS: should have empty repo registry loaded into env" '
+    grep -F "REPO_REGISTRY" out &&
+    grep -xF "REPO_REGISTRY: []" out
+'
+
 test_expect_success NO_REPOS \
     "NO_REPOS: should not have repository data" '
     for repo in sourcecred/example-git sourcecred/example-github; do

src/homepage/server.js

@@ -14,6 +14,9 @@ import {createRoutes} from "./createRoutes";
 import {resolveRouteFromPath, resolveTitleFromPath} from "./routeData";
 import dedent from "../util/dedent";
 
+// Side effect for testing purposes
+console.log(`REPO_REGISTRY: ${process.env.REPO_REGISTRY || "bad"}`);
+
 export default function render(
   locals: {+path: string, +assets: {[string]: string}},
   callback: (error: ?mixed, result?: string) => void