Introduce transforms pipeline

Summary:
public

This diff introduces an internal transforms pipeline that integrates with the external one. This has been a feature we've been looking to implement for a long time to use babel instead of `replace` with regexps on many parts of the packager.

Also, to split the bundle we'll need to run one transform. Internally for Facebook we can run the system-import transform altogether withe the other ones. For OSS we offer `transformer.js` which people can use out of the box if they're writing ES6 code. For those people, `transformer.js` will also run the internal transforms`. However they might want to tune the transforms, or even write the code on another language that compiles to Javascript and use a complete different transformer. On those cases we'll need to run the external transforms first and pipe the output through the internal transforms. Note that the order it's important as the internal transforms assume the code is written in JS, though the original code could be on other scripting languages (CoffeeScript, TypeScript, etc).

Reviewed By: davidaurelio

Differential Revision: D2725109

fb-gh-sync-id: d764e209c78743419c4cb97068495c771372ab90

local-cli/server/runServer.js

@@ -68,6 +68,7 @@ function getAppMiddleware(args, config) {
     cacheVersion: '3',
     getTransformOptionsModulePath: config.getTransformOptionsModulePath,
     transformModulePath: transformerPath,
+    enableInternalTransforms: args['enable-internal-transforms'],
     assetRoots: args.assetRoots,
     assetExts: ['png', 'jpg', 'jpeg', 'bmp', 'gif', 'webp'],
     resetCache: args.resetCache || args['reset-cache'],

local-cli/server/server.js

@@ -53,6 +53,10 @@ function _server(argv, config, resolve, reject) {
     type: 'string',
     default: require.resolve('../../packager/transformer'),
     description: 'Specify a custom transformer to be used (absolute path)'
+  }, {
+    command: 'enable-internal-transforms',
+    type: 'boolean',
+    default: false
   }, {
     command: 'resetCache',
     description: 'Removes cached files',

packager/react-packager/index.js

@@ -15,6 +15,7 @@ useGracefulFs();
 var debug = require('debug');
 var omit = require('underscore').omit;
 var Activity = require('./src/Activity');
+var Transforms = require('./src/transforms');
 
 exports.createServer = createServer;
 exports.middleware = function(options) {
@@ -23,6 +24,7 @@ exports.middleware = function(options) {
 };
 
 exports.Activity = Activity;
+exports.getTransforms = Transforms.getAll;
 
 // Renamed "package" to "bundle". But maintain backwards
 // compat.

packager/react-packager/src/Bundler/index.js

@@ -56,6 +56,10 @@ const validateOpts = declareOpts({
     type:'string',
     required: false,
   },
+  enableInternalTransforms: {
+    type: 'boolean',
+    default: false,
+  },
   nonPersistent: {
     type: 'boolean',
     default: false,
@@ -131,6 +135,7 @@ class Bundler {
       blacklistRE: opts.blacklistRE,
       cache: this._cache,
       transformModulePath: opts.transformModulePath,
+      enableInternalTransforms: opts.enableInternalTransforms,
     });
 
     this._projectRoots = opts.projectRoots;

packager/react-packager/src/JSTransformer/index.js

@@ -52,7 +52,11 @@ const validateOpts = declareOpts({
   transformTimeoutInterval: {
     type: 'number',
     default: DEFAULT_MAX_CALL_TIME,
-  }
+  },
+  enableInternalTransforms: {
+    type: 'boolean',
+    default: false,
+  },
 });
 
 class Transformer {
@@ -60,6 +64,8 @@ class Transformer {
     const opts = this._opts = validateOpts(options);
 
     this._cache = opts.cache;
+    this._transformModulePath = opts.transformModulePath;
+    this._enableInternalTransforms = opts.enableInternalTransforms;
 
     if (opts.transformModulePath != null) {
       this._workers = workerFarm({
@@ -68,7 +74,7 @@ class Transformer {
         maxCallsPerWorker: MAX_CALLS_PER_WORKER,
         maxCallTime: opts.transformTimeoutInterval,
         maxRetries: MAX_RETRIES,
-      }, opts.transformModulePath);
+      }, require.resolve('./worker'));
 
       this._transform = Promise.denodeify(this._workers);
     }
@@ -102,7 +108,11 @@ class Transformer {
           return this._transform({
             sourceCode,
             filename: filePath,
-            options,
+            options: {
+              ...options,
+              externalTransformModulePath: this._transformModulePath,
+              enableInternalTransforms: this._enableInternalTransforms,
+            },
           }).then(res => {
             if (res.error) {
               console.warn(

packager/react-packager/src/JSTransformer/worker.js

@@ -8,27 +8,66 @@
  */
 'use strict';
 
-var transformer = require('./transformer');
+var babel = require('babel-core');
+var Transforms = require('../transforms');
 
-module.exports = function (data, callback) {
+// Runs internal transforms on the given sourceCode. Note that internal
+// transforms should be run after the external ones to ensure that they run on
+// Javascript code
+function internalTransforms(sourceCode, filename, options) {
+  var result = babel.transform(sourceCode, {
+    retainLines: true,
+    compact: true,
+    comments: false,
+    filename: filename,
+    sourceFileName: filename,
+    sourceMaps: false,
+    plugins: Transforms.getAll()
+  });
+
+  return {
+    code: result.code,
+    filename: filename,
+  };
+}
+
+function onExternalTransformDone(data, callback, error, externalOutput) {
   var result;
-  try {
-    result = transformer.transform(
-      data.transformSets,
-      data.sourceCode,
+  if (data.options.enableInternalTransforms) {
+    result = internalTransforms(
+      externalOutput.code,
+      externalOutput.filename,
       data.options
     );
-  } catch (e) {
-    return callback(null, {
-      error: {
-        lineNumber: e.lineNumber,
-        column: e.column,
-        message: e.message,
-        stack: e.stack,
-        description: e.description
-      }
-    });
+  } else {
+    result = externalOutput;
   }
 
   callback(null, result);
+}
+
+module.exports = function(data, callback) {
+  try {
+    if (data.options.externalTransformModulePath) {
+      var externalTransformModule = require(
+        data.options.externalTransformModulePath
+      );
+      externalTransformModule(
+        data,
+        onExternalTransformDone.bind(null, data, callback)
+      );
+    } else {
+      onExternalTransformDone(
+        data,
+        callback,
+        null,
+        {
+          code: data.sourceCode,
+          filename: data.filename
+        }
+      );
+    }
+  } catch (e) {
+    callback(e);
+  }
 };

packager/react-packager/src/Server/index.js

@@ -52,6 +52,10 @@ const validateOpts = declareOpts({
     type: 'boolean',
     default: false,
   },
+  enableInternalTransforms: {
+    type: 'boolean',
+    default: false,
+  },
   assetRoots: {
     type: 'array',
     required: false,

packager/react-packager/src/SocketInterface/index.js

@@ -30,7 +30,9 @@ const SocketInterface = {
         const value = options[key];
         if (value) {
           hash.update(
-            typeof value === 'string' ? value : JSON.stringify(value)
+            options[key] != null && typeof value === 'string'
+              ? value
+              : JSON.stringify(value)
           );
         }
       });

packager/react-packager/src/transforms/index.js

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+exports.getAll = function() {
+  return [
+    require('./babel-plugin-system-import'),
+  ];
+};
+

packager/transformer.js

@@ -15,6 +15,7 @@ const fs = require('fs');
 const inlineRequires = require('fbjs-scripts/babel-6/inline-requires');
 const json5 = require('json5');
 const path = require('path');
+const ReactPackager = require('./react-packager');
 
 const babelRC =
   json5.parse(
@@ -53,10 +54,13 @@ function transform(src, filename, options) {
     return plugin;
   });
 
+  config.plugins = config.plugins.concat(ReactPackager.getTransforms());
+
   const result = babel.transform(src, Object.assign({}, babelRC, config));
 
   return {
-    code: result.code
+    code: result.code,
+    filename: filename,
   };
 }