From 274007c90d9fb785765e571c415bd0a158d40934 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Sun, 18 Mar 2018 22:43:23 -0700 Subject: [PATCH] Configure Webpack for backend applications (#84) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Running `yarn backend` will now bundle backend applications. They’ll be placed into the new `bin/` directory. This enables us to use ES6 modules with the standard syntax, Flow types, and all the other goodies that we’ve come to expect. A backend build takes about 2.5s on my laptop. Created by forking the prod configuration to a backend configuration and trimming it down appropriately. To test out the new changes, this commit changes `fetchGitHubRepo` and its driver to use the ES6 module system and Flow types, both of which are properly resolved. Test Plan: Run `yarn backend`. Then, you can directly run an entry point via ``` $ node bin/fetchAndPrintGitHubRepo.js sourcecred example-repo "${TOKEN}" ``` or invoke the standard test driver via ```shell $ GITHUB_TOKEN="${TOKEN}" src/backend/fetchGitHubRepoTest.sh ``` where `${TOKEN}` is your GitHub authentication token. wchargin-branch: webpack-backend --- .gitignore | 3 + .prettierignore | 1 + config/paths.js | 11 ++ config/webpack.config.backend.js | 83 +++++++++++++ package.json | 1 + scripts/backend.js | 130 +++++++++++++++++++++ src/backend/bin/fetchAndPrintGitHubRepo.js | 4 +- src/backend/fetchGitHubRepo.js | 11 +- src/backend/fetchGitHubRepoTest.sh | 7 +- 9 files changed, 245 insertions(+), 6 deletions(-) create mode 100644 config/webpack.config.backend.js create mode 100644 scripts/backend.js diff --git a/.gitignore b/.gitignore index 2785b06..a73d507 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ # production /build +# backend +/bin + # misc .env.local .env.development.local diff --git a/.prettierignore b/.prettierignore index c6417d9..33791f1 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ +bin flow-typed diff --git a/config/paths.js b/config/paths.js index af6e3cd..7a1714a 100644 --- a/config/paths.js +++ b/config/paths.js @@ -52,4 +52,15 @@ module.exports = { appNodeModules: resolveApp("node_modules"), publicUrl: getPublicUrl(resolveApp("package.json")), servedPath: getServedPath(resolveApp("package.json")), + + backendBuild: resolveApp("bin"), + // This object should have one key-value pair per entry point. For + // each key, the value should be the path to the entry point for the + // source file, and the key will be the filename of the bundled entry + // point within the build directory. + backendEntryPoints: { + fetchAndPrintGitHubRepo: resolveApp( + "src/backend/bin/fetchAndPrintGitHubRepo.js" + ), + }, }; diff --git a/config/webpack.config.backend.js b/config/webpack.config.backend.js new file mode 100644 index 0000000..e9c2c28 --- /dev/null +++ b/config/webpack.config.backend.js @@ -0,0 +1,83 @@ +"use strict"; + +const path = require("path"); +const webpack = require("webpack"); +const eslintFormatter = require("react-dev-utils/eslintFormatter"); +const ModuleScopePlugin = require("react-dev-utils/ModuleScopePlugin"); +const paths = require("./paths"); + +// This is the backend configuration. It builds applications that target +// Node and will not run in a browser. +module.exports = { + // Don't attempt to continue if there are any errors. + bail: true, + // Target Node instead of the browser. + target: "node", + entry: paths.backendEntryPoints, + output: { + path: paths.backendBuild, + // Generated JS file names (with nested folders). + // There will be one main bundle, and one file per asynchronous chunk. + // We don't currently advertise code splitting but Webpack supports it. + filename: "[name].js", + chunkFilename: "[name].[chunkhash:8].chunk.js", + }, + resolve: { + extensions: [".js", ".json"], + plugins: [ + // Prevents users from importing files from outside of src/ (or node_modules/). + // This often causes confusion because we only process files within src/ with babel. + // To fix this, we prevent you from importing files out of src/ -- if you'd like to, + // please link the files into your node_modules/ and let module-resolution kick in. + // Make sure your source files are compiled, as they will not be processed in any way. + new ModuleScopePlugin(paths.appSrc, [paths.appPackageJson]), + ], + }, + module: { + strictExportPresence: true, + rules: [ + // First, run the linter. + // It's important to do this before Babel processes the JS. + { + test: /\.(js|jsx|mjs)$/, + enforce: "pre", + use: [ + { + options: { + formatter: eslintFormatter, + eslintPath: require.resolve("eslint"), + }, + loader: require.resolve("eslint-loader"), + }, + ], + include: paths.appSrc, + }, + { + // "oneOf" will traverse all following loaders until one will + // match the requirements. If no loader matches, it will fail. + oneOf: [ + // Process JS with Babel. + { + test: /\.(js|jsx|mjs)$/, + include: paths.appSrc, + loader: require.resolve("babel-loader"), + options: { + compact: true, + }, + }, + ], + }, + ], + }, + plugins: [ + new webpack.DefinePlugin({ + "process.env.NODE_ENV": JSON.stringify( + process.env.NODE_ENV || "development" + ), + }), + // See: + // - https://github.com/andris9/encoding/issues/16 (the culprit) + // - https://github.com/bitinn/node-fetch/issues/41 (the solution) + new webpack.IgnorePlugin(/\/iconv-loader$/), + ], +}; diff --git a/package.json b/package.json index ae7f6fe..70f923d 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "check-pretty": "prettier --list-different '**/*.js'", "start": "node scripts/start.js", "build": "node scripts/build.js", + "backend": "node scripts/backend.js", "test": "node scripts/test.js --env=jsdom", "flow": "flow", "travis": "npm run check-pretty && npm run flow && CI=true npm run test" diff --git a/scripts/backend.js b/scripts/backend.js new file mode 100644 index 0000000..4b96d38 --- /dev/null +++ b/scripts/backend.js @@ -0,0 +1,130 @@ +"use strict"; + +// Do this as the first thing so that any code reading it knows the right env. +process.env.NODE_ENV = process.env.NODE_ENV || "development"; +process.env.BABEL_ENV = process.env.NODE_ENV; + +// Makes the script crash on unhandled rejections instead of silently +// ignoring them. In the future, promise rejections that are not handled will +// terminate the Node.js process with a non-zero exit code. +process.on("unhandledRejection", (err) => { + throw err; +}); + +// Ensure environment variables are read. +require("../config/env"); + +const path = require("path"); +const chalk = require("chalk"); +const fs = require("fs-extra"); +const webpack = require("webpack"); +const config = require("../config/webpack.config.backend"); +const paths = require("../config/paths"); +const checkRequiredFiles = require("react-dev-utils/checkRequiredFiles"); +const formatWebpackMessages = require("react-dev-utils/formatWebpackMessages"); +const FileSizeReporter = require("react-dev-utils/FileSizeReporter"); +const printBuildError = require("react-dev-utils/printBuildError"); + +const measureFileSizesBeforeBuild = + FileSizeReporter.measureFileSizesBeforeBuild; +const printFileSizesAfterBuild = FileSizeReporter.printFileSizesAfterBuild; + +// These sizes are pretty large. We'll warn for bundles exceeding them. +const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024; +const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024; + +// Warn and crash if required files are missing +if (!checkRequiredFiles(Object.values(paths.backendEntryPoints))) { + process.exit(1); +} + +// First, read the current file sizes in build directory. +// This lets us display how much they changed later. +measureFileSizesBeforeBuild(paths.backendBuild) + .then((previousFileSizes) => { + // Remove all content but keep the directory so that + // if you're in it, you don't end up in Trash + fs.emptyDirSync(paths.backendBuild); + // Start the webpack build + return build(previousFileSizes); + }) + .then( + ({stats, previousFileSizes, warnings}) => { + if (warnings.length) { + console.log(chalk.yellow("Compiled with warnings.\n")); + console.log(warnings.join("\n\n")); + console.log( + "\nSearch for the " + + chalk.underline(chalk.yellow("keywords")) + + " to learn more about each warning." + ); + console.log( + "To ignore, add " + + chalk.cyan("// eslint-disable-next-line") + + " to the line before.\n" + ); + } else { + console.log(chalk.green("Compiled successfully.\n")); + } + + console.log("File sizes after gzip:\n"); + printFileSizesAfterBuild( + stats, + previousFileSizes, + paths.backendBuild, + WARN_AFTER_BUNDLE_GZIP_SIZE, + WARN_AFTER_CHUNK_GZIP_SIZE + ); + console.log(); + + const buildFolder = path.relative(process.cwd(), paths.backendBuild); + console.log(`Build completed; results in '${buildFolder}'.`); + }, + (err) => { + console.log(chalk.red("Failed to compile.\n")); + printBuildError(err); + process.exit(1); + } + ); + +// Create the backend build +function build(previousFileSizes) { + console.log("Building backend applications..."); + + let compiler = webpack(config); + return new Promise((resolve, reject) => { + compiler.run((err, stats) => { + if (err) { + return reject(err); + } + const messages = formatWebpackMessages(stats.toJson({}, true)); + if (messages.errors.length) { + // Only keep the first error. Others are often indicative + // of the same problem, but confuse the reader with noise. + if (messages.errors.length > 1) { + messages.errors.length = 1; + } + return reject(new Error(messages.errors.join("\n\n"))); + } + if ( + process.env.CI && + (typeof process.env.CI !== "string" || + process.env.CI.toLowerCase() !== "false") && + messages.warnings.length + ) { + console.log( + chalk.yellow( + "\nTreating warnings as errors because process.env.CI = true.\n" + + "Most CI servers set it automatically.\n" + ) + ); + return reject(new Error(messages.warnings.join("\n\n"))); + } + return resolve({ + stats, + previousFileSizes, + warnings: messages.warnings, + }); + }); + }); +} diff --git a/src/backend/bin/fetchAndPrintGitHubRepo.js b/src/backend/bin/fetchAndPrintGitHubRepo.js index b36ec83..41493f3 100644 --- a/src/backend/bin/fetchAndPrintGitHubRepo.js +++ b/src/backend/bin/fetchAndPrintGitHubRepo.js @@ -11,8 +11,8 @@ * from https://github.com/settings/tokens/new. */ -const fetchGitHubRepo = require("../fetchGitHubRepo"); -const stringify = require("json-stable-stringify"); +import fetchGitHubRepo from "../fetchGitHubRepo"; +import stringify from "json-stable-stringify"; function parseArgs() { const argv = process.argv.slice(2); diff --git a/src/backend/fetchGitHubRepo.js b/src/backend/fetchGitHubRepo.js index 22e303a..e3a46f4 100644 --- a/src/backend/fetchGitHubRepo.js +++ b/src/backend/fetchGitHubRepo.js @@ -1,9 +1,10 @@ +// @flow /* * API to scrape data from a GitHub repo using the GitHub API. See the * docstring of the default export for more details. */ -const fetch = require("node-fetch"); +import fetch from "node-fetch"; /** * Scrape data from a GitHub repo using the GitHub API. @@ -20,7 +21,11 @@ const fetch = require("node-fetch"); * scraped from the repository, with data format to be specified * later */ -module.exports = function fetchGitHubRepo(repoOwner, repoName, token) { +export default function fetchGitHubRepo( + repoOwner: string, + repoName: string, + token: string +): Promise { repoOwner = String(repoOwner); repoName = String(repoName); token = String(token); @@ -138,7 +143,7 @@ module.exports = function fetchGitHubRepo(repoOwner, repoName, token) { const variables = {repoOwner, repoName}; const payload = {query, variables}; return postQuery(payload, token); -}; +} const GITHUB_GRAPHQL_SERVER = "https://api.github.com/graphql"; diff --git a/src/backend/fetchGitHubRepoTest.sh b/src/backend/fetchGitHubRepoTest.sh index 5b800dd..44d8e16 100755 --- a/src/backend/fetchGitHubRepoTest.sh +++ b/src/backend/fetchGitHubRepoTest.sh @@ -3,13 +3,18 @@ set -eu main() { + if ! [ -d bin ]; then + printf >&2 'Backend applications have not been built.\n' + printf >&2 'Please run "yarn backend".\n' + return 1 + fi if [ -z "${GITHUB_TOKEN:-}" ]; then printf >&2 'Please set the GITHUB_TOKEN environment variable\n' printf >&2 'to a 40-character hex string API token from GitHub.\n' return 1 fi output="$(mktemp)" - node src/backend/bin/fetchAndPrintGitHubRepo.js \ + node bin/fetchAndPrintGitHubRepo.js \ sourcecred example-repo "${GITHUB_TOKEN}" \ >"${output}" \ ;