Configure Webpack for backend applications (#84)

Summary:
Running `yarn backend` will now bundle backend applications. They’ll be
placed into the new `bin/` directory. This enables us to use ES6 modules
with the standard syntax, Flow types, and all the other goodies that
we’ve come to expect. A backend build takes about 2.5s on my laptop.

Created by forking the prod configuration to a backend configuration and
trimming it down appropriately.

To test out the new changes, this commit changes `fetchGitHubRepo` and
its driver to use the ES6 module system and Flow types, both of which
are properly resolved.

Test Plan:
Run `yarn backend`. Then, you can directly run an entry point via
```
$ node bin/fetchAndPrintGitHubRepo.js sourcecred example-repo "${TOKEN}"
```
or invoke the standard test driver via
```shell
$ GITHUB_TOKEN="${TOKEN}" src/backend/fetchGitHubRepoTest.sh
```
where `${TOKEN}` is your GitHub authentication token.

wchargin-branch: webpack-backend

.gitignore

@@ -9,6 +9,9 @@
 # production
 /build
 
+# backend
+/bin
+
 # misc
 .env.local
 .env.development.local

.prettierignore

@@ -1 +1,2 @@
+bin
 flow-typed

config/paths.js

@@ -52,4 +52,15 @@ module.exports = {
   appNodeModules: resolveApp("node_modules"),
   publicUrl: getPublicUrl(resolveApp("package.json")),
   servedPath: getServedPath(resolveApp("package.json")),
+
+  backendBuild: resolveApp("bin"),
+  // This object should have one key-value pair per entry point. For
+  // each key, the value should be the path to the entry point for the
+  // source file, and the key will be the filename of the bundled entry
+  // point within the build directory.
+  backendEntryPoints: {
+    fetchAndPrintGitHubRepo: resolveApp(
+      "src/backend/bin/fetchAndPrintGitHubRepo.js"
+    ),
+  },
 };

config/webpack.config.backend.js

@@ -0,0 +1,83 @@
+"use strict";
+
+const path = require("path");
+const webpack = require("webpack");
+const eslintFormatter = require("react-dev-utils/eslintFormatter");
+const ModuleScopePlugin = require("react-dev-utils/ModuleScopePlugin");
+const paths = require("./paths");
+
+// This is the backend configuration. It builds applications that target
+// Node and will not run in a browser.
+module.exports = {
+  // Don't attempt to continue if there are any errors.
+  bail: true,
+  // Target Node instead of the browser.
+  target: "node",
+  entry: paths.backendEntryPoints,
+  output: {
+    path: paths.backendBuild,
+    // Generated JS file names (with nested folders).
+    // There will be one main bundle, and one file per asynchronous chunk.
+    // We don't currently advertise code splitting but Webpack supports it.
+    filename: "[name].js",
+    chunkFilename: "[name].[chunkhash:8].chunk.js",
+  },
+  resolve: {
+    extensions: [".js", ".json"],
+    plugins: [
+      // Prevents users from importing files from outside of src/ (or node_modules/).
+      // This often causes confusion because we only process files within src/ with babel.
+      // To fix this, we prevent you from importing files out of src/ -- if you'd like to,
+      // please link the files into your node_modules/ and let module-resolution kick in.
+      // Make sure your source files are compiled, as they will not be processed in any way.
+      new ModuleScopePlugin(paths.appSrc, [paths.appPackageJson]),
+    ],
+  },
+  module: {
+    strictExportPresence: true,
+    rules: [
+      // First, run the linter.
+      // It's important to do this before Babel processes the JS.
+      {
+        test: /\.(js|jsx|mjs)$/,
+        enforce: "pre",
+        use: [
+          {
+            options: {
+              formatter: eslintFormatter,
+              eslintPath: require.resolve("eslint"),
+            },
+            loader: require.resolve("eslint-loader"),
+          },
+        ],
+        include: paths.appSrc,
+      },
+      {
+        // "oneOf" will traverse all following loaders until one will
+        // match the requirements. If no loader matches, it will fail.
+        oneOf: [
+          // Process JS with Babel.
+          {
+            test: /\.(js|jsx|mjs)$/,
+            include: paths.appSrc,
+            loader: require.resolve("babel-loader"),
+            options: {
+              compact: true,
+            },
+          },
+        ],
+      },
+    ],
+  },
+  plugins: [
+    new webpack.DefinePlugin({
+      "process.env.NODE_ENV": JSON.stringify(
+        process.env.NODE_ENV || "development"
+      ),
+    }),
+    // See:
+    //   - https://github.com/andris9/encoding/issues/16 (the culprit)
+    //   - https://github.com/bitinn/node-fetch/issues/41 (the solution)
+    new webpack.IgnorePlugin(/\/iconv-loader$/),
+  ],
+};

package.json

@@ -57,6 +57,7 @@
     "check-pretty": "prettier --list-different '**/*.js'",
     "start": "node scripts/start.js",
     "build": "node scripts/build.js",
+    "backend": "node scripts/backend.js",
     "test": "node scripts/test.js --env=jsdom",
     "flow": "flow",
     "travis": "npm run check-pretty && npm run flow && CI=true npm run test"

scripts/backend.js

@@ -0,0 +1,130 @@
+"use strict";
+
+// Do this as the first thing so that any code reading it knows the right env.
+process.env.NODE_ENV = process.env.NODE_ENV || "development";
+process.env.BABEL_ENV = process.env.NODE_ENV;
+
+// Makes the script crash on unhandled rejections instead of silently
+// ignoring them. In the future, promise rejections that are not handled will
+// terminate the Node.js process with a non-zero exit code.
+process.on("unhandledRejection", (err) => {
+  throw err;
+});
+
+// Ensure environment variables are read.
+require("../config/env");
+
+const path = require("path");
+const chalk = require("chalk");
+const fs = require("fs-extra");
+const webpack = require("webpack");
+const config = require("../config/webpack.config.backend");
+const paths = require("../config/paths");
+const checkRequiredFiles = require("react-dev-utils/checkRequiredFiles");
+const formatWebpackMessages = require("react-dev-utils/formatWebpackMessages");
+const FileSizeReporter = require("react-dev-utils/FileSizeReporter");
+const printBuildError = require("react-dev-utils/printBuildError");
+
+const measureFileSizesBeforeBuild =
+  FileSizeReporter.measureFileSizesBeforeBuild;
+const printFileSizesAfterBuild = FileSizeReporter.printFileSizesAfterBuild;
+
+// These sizes are pretty large. We'll warn for bundles exceeding them.
+const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
+const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024;
+
+// Warn and crash if required files are missing
+if (!checkRequiredFiles(Object.values(paths.backendEntryPoints))) {
+  process.exit(1);
+}
+
+// First, read the current file sizes in build directory.
+// This lets us display how much they changed later.
+measureFileSizesBeforeBuild(paths.backendBuild)
+  .then((previousFileSizes) => {
+    // Remove all content but keep the directory so that
+    // if you're in it, you don't end up in Trash
+    fs.emptyDirSync(paths.backendBuild);
+    // Start the webpack build
+    return build(previousFileSizes);
+  })
+  .then(
+    ({stats, previousFileSizes, warnings}) => {
+      if (warnings.length) {
+        console.log(chalk.yellow("Compiled with warnings.\n"));
+        console.log(warnings.join("\n\n"));
+        console.log(
+          "\nSearch for the " +
+            chalk.underline(chalk.yellow("keywords")) +
+            " to learn more about each warning."
+        );
+        console.log(
+          "To ignore, add " +
+            chalk.cyan("// eslint-disable-next-line") +
+            " to the line before.\n"
+        );
+      } else {
+        console.log(chalk.green("Compiled successfully.\n"));
+      }
+
+      console.log("File sizes after gzip:\n");
+      printFileSizesAfterBuild(
+        stats,
+        previousFileSizes,
+        paths.backendBuild,
+        WARN_AFTER_BUNDLE_GZIP_SIZE,
+        WARN_AFTER_CHUNK_GZIP_SIZE
+      );
+      console.log();
+
+      const buildFolder = path.relative(process.cwd(), paths.backendBuild);
+      console.log(`Build completed; results in '${buildFolder}'.`);
+    },
+    (err) => {
+      console.log(chalk.red("Failed to compile.\n"));
+      printBuildError(err);
+      process.exit(1);
+    }
+  );
+
+// Create the backend build
+function build(previousFileSizes) {
+  console.log("Building backend applications...");
+
+  let compiler = webpack(config);
+  return new Promise((resolve, reject) => {
+    compiler.run((err, stats) => {
+      if (err) {
+        return reject(err);
+      }
+      const messages = formatWebpackMessages(stats.toJson({}, true));
+      if (messages.errors.length) {
+        // Only keep the first error. Others are often indicative
+        // of the same problem, but confuse the reader with noise.
+        if (messages.errors.length > 1) {
+          messages.errors.length = 1;
+        }
+        return reject(new Error(messages.errors.join("\n\n")));
+      }
+      if (
+        process.env.CI &&
+        (typeof process.env.CI !== "string" ||
+          process.env.CI.toLowerCase() !== "false") &&
+        messages.warnings.length
+      ) {
+        console.log(
+          chalk.yellow(
+            "\nTreating warnings as errors because process.env.CI = true.\n" +
+              "Most CI servers set it automatically.\n"
+          )
+        );
+        return reject(new Error(messages.warnings.join("\n\n")));
+      }
+      return resolve({
+        stats,
+        previousFileSizes,
+        warnings: messages.warnings,
+      });
+    });
+  });
+}

src/backend/bin/fetchAndPrintGitHubRepo.js

@@ -11,8 +11,8 @@
  * from https://github.com/settings/tokens/new.
  */
 
-const fetchGitHubRepo = require("../fetchGitHubRepo");
+import fetchGitHubRepo from "../fetchGitHubRepo";
-const stringify = require("json-stable-stringify");
+import stringify from "json-stable-stringify";
 
 function parseArgs() {
   const argv = process.argv.slice(2);

src/backend/fetchGitHubRepo.js

@@ -1,9 +1,10 @@
+// @flow
 /*
  * API to scrape data from a GitHub repo using the GitHub API. See the
  * docstring of the default export for more details.
  */
 
-const fetch = require("node-fetch");
+import fetch from "node-fetch";
 
 /**
  * Scrape data from a GitHub repo using the GitHub API.
@@ -20,7 +21,11 @@ const fetch = require("node-fetch");
  *    scraped from the repository, with data format to be specified
  *    later
  */
-module.exports = function fetchGitHubRepo(repoOwner, repoName, token) {
+export default function fetchGitHubRepo(
+  repoOwner: string,
+  repoName: string,
+  token: string
+): Promise<Object> {
   repoOwner = String(repoOwner);
   repoName = String(repoName);
   token = String(token);
@@ -138,7 +143,7 @@ module.exports = function fetchGitHubRepo(repoOwner, repoName, token) {
   const variables = {repoOwner, repoName};
   const payload = {query, variables};
   return postQuery(payload, token);
-};
+}
 
 const GITHUB_GRAPHQL_SERVER = "https://api.github.com/graphql";
 

src/backend/fetchGitHubRepoTest.sh

@@ -3,13 +3,18 @@
 set -eu
 
 main() {
+  if ! [ -d bin ]; then
+    printf >&2 'Backend applications have not been built.\n'
+    printf >&2 'Please run "yarn backend".\n'
+    return 1
+  fi
   if [ -z "${GITHUB_TOKEN:-}" ]; then
     printf >&2 'Please set the GITHUB_TOKEN environment variable\n'
     printf >&2 'to a 40-character hex string API token from GitHub.\n'
     return 1
   fi
   output="$(mktemp)"
-  node src/backend/bin/fetchAndPrintGitHubRepo.js \
+  node bin/fetchAndPrintGitHubRepo.js \
     sourcecred example-repo "${GITHUB_TOKEN}" \
     >"${output}" \
     ;