From bfecccd180ea9a60a51b3bc547b6356aad90c5ba Mon Sep 17 00:00:00 2001 From: Rafael Oleza Date: Fri, 16 Mar 2018 01:36:12 -0700 Subject: [PATCH] Avoid parsing JSON files with babel Summary: Metro does not need to use babel to parse JSON files, it only needs to wrap the stringified JSON directly into the `define` function. This makes the transformation of JSON files much faster, plus prevent out of memory exceptions when having huge JSON files. This diff fixes https://github.com/facebook/metro/issues/146 Reviewed By: mjesun Differential Revision: D7227095 fbshipit-source-id: 5d1a9cb2d1c7162a403c00dc43e46f781fbd1514 --- .../metro/src/JSTransformer/worker/index.js | 185 +++++++++--------- .../metro/src/ModuleGraph/test-helpers.js | 2 +- .../src/ModuleGraph/worker/JsFileWrapping.js | 11 +- .../worker/__tests__/JsFileWrapping-test.js | 35 ++++ .../worker/__tests__/transform-module-test.js | 2 +- .../ModuleGraph/worker/transform-module.js | 4 +- 6 files changed, 135 insertions(+), 104 deletions(-) diff --git a/packages/metro/src/JSTransformer/worker/index.js b/packages/metro/src/JSTransformer/worker/index.js index 128210c5..a5e744d7 100644 --- a/packages/metro/src/JSTransformer/worker/index.js +++ b/packages/metro/src/JSTransformer/worker/index.js @@ -94,30 +94,91 @@ export type Data = { transformFileEndLogEntry: LogEntry, }; -function postTransform( +function getDynamicDepsBehavior( + inPackages: DynamicRequiresBehavior, + filename: string, +): DynamicRequiresBehavior { + switch (inPackages) { + case 'reject': + return 'reject'; + case 'throwAtRuntime': + const isPackage = /(?:^|[/\\])node_modules[/\\]/.test(filename); + return isPackage ? inPackages : 'reject'; + default: + (inPackages: empty); + throw new Error( + `invalid value for dynamic deps behavior: \`${inPackages}\``, + ); + } +} + +async function transformCode( filename: string, localPath: LocalPath, sourceCode: string, + transformerPath: string, isScript: boolean, options: Options, - transformFileStartLogEntry: LogEntry, + assetExts: $ReadOnlyArray, + assetRegistryPath: string, asyncRequireModulePath: string, dynamicDepsInPackages: DynamicRequiresBehavior, - receivedAst: ?Ast, -): Data { +): Promise { + const transformFileStartLogEntry = { + action_name: 'Transforming file', + action_phase: 'start', + file_name: filename, + log_entry_label: 'Transforming file', + start_timestamp: process.hrtime(), + }; + + if (filename.endsWith('.json')) { + const code = JsFileWrapping.wrapJson(sourceCode); + + const transformFileEndLogEntry = getEndLogEntry( + transformFileStartLogEntry, + filename, + ); + + return { + result: {dependencies: [], code, map: []}, + transformFileStartLogEntry, + transformFileEndLogEntry, + }; + } + + const plugins = options.dev + ? [] + : [[inlinePlugin, options], [constantFoldingPlugin, options]]; + + // $FlowFixMe TODO t26372934 Plugin system + const transformer: Transformer<*> = require(transformerPath); + + const transformerArgs = { + filename, + localPath, + options, + plugins, + src: sourceCode, + }; + + const transformResult = isAsset(filename, assetExts) + ? await assetTransformer.transform( + transformerArgs, + assetRegistryPath, + options.assetDataPlugins, + ) + : await transformer.transform(transformerArgs); + // Transformers can ouptut null ASTs (if they ignore the file). In that case // we need to parse the module source code to get their AST. - const ast = receivedAst || babylon.parse(sourceCode, {sourceType: 'module'}); + const ast = + transformResult.ast || babylon.parse(sourceCode, {sourceType: 'module'}); - const timeDelta = process.hrtime(transformFileStartLogEntry.start_timestamp); - const duration_ms = Math.round((timeDelta[0] * 1e9 + timeDelta[1]) / 1e6); - const transformFileEndLogEntry = { - action_name: 'Transforming file', - action_phase: 'end', - file_name: filename, - duration_ms, - log_entry_label: 'Transforming file', - }; + const transformFileEndLogEntry = getEndLogEntry( + transformFileStartLogEntry, + filename, + ); let dependencies, wrappedAst; @@ -185,89 +246,6 @@ function postTransform( }; } -function getDynamicDepsBehavior( - inPackages: DynamicRequiresBehavior, - filename: string, -): DynamicRequiresBehavior { - switch (inPackages) { - case 'reject': - return 'reject'; - case 'throwAtRuntime': - const isPackage = /(?:^|[/\\])node_modules[/\\]/.test(filename); - return isPackage ? inPackages : 'reject'; - default: - (inPackages: empty); - throw new Error( - `invalid value for dynamic deps behavior: \`${inPackages}\``, - ); - } -} - -function transformCode( - filename: string, - localPath: LocalPath, - sourceCode: string, - transformerPath: string, - isScript: boolean, - options: Options, - assetExts: $ReadOnlyArray, - assetRegistryPath: string, - asyncRequireModulePath: string, - dynamicDepsInPackages: DynamicRequiresBehavior, -): Data | Promise { - const isJson = filename.endsWith('.json'); - - if (isJson) { - sourceCode = 'module.exports=' + sourceCode; - } - - const transformFileStartLogEntry = { - action_name: 'Transforming file', - action_phase: 'start', - file_name: filename, - log_entry_label: 'Transforming file', - start_timestamp: process.hrtime(), - }; - - const plugins = options.dev - ? [] - : [[inlinePlugin, options], [constantFoldingPlugin, options]]; - - // $FlowFixMe TODO t26372934 Plugin system - const transformer: Transformer<*> = require(transformerPath); - - const transformerArgs = { - filename, - localPath, - options, - plugins, - src: sourceCode, - }; - - const transformResult = isAsset(filename, assetExts) - ? assetTransformer.transform( - transformerArgs, - assetRegistryPath, - options.assetDataPlugins, - ) - : transformer.transform(transformerArgs); - - const postTransformArgs = [ - filename, - localPath, - sourceCode, - isScript, - options, - transformFileStartLogEntry, - asyncRequireModulePath, - dynamicDepsInPackages, - ]; - - return transformResult instanceof Promise - ? transformResult.then(({ast}) => postTransform(...postTransformArgs, ast)) - : postTransform(...postTransformArgs, transformResult.ast); -} - function minifyCode( filename: string, code: string, @@ -292,6 +270,19 @@ function isAsset(filePath: string, assetExts: $ReadOnlyArray): boolean { return assetExts.indexOf(path.extname(filePath).slice(1)) !== -1; } +function getEndLogEntry(startLogEntry: LogEntry, filename: string): LogEntry { + const timeDelta = process.hrtime(startLogEntry.start_timestamp); + const duration_ms = Math.round((timeDelta[0] * 1e9 + timeDelta[1]) / 1e6); + + return { + action_name: 'Transforming file', + action_phase: 'end', + file_name: filename, + duration_ms, + log_entry_label: 'Transforming file', + }; +} + class InvalidRequireCallError extends Error { innerError: collectDependencies.InvalidRequireCallError; filename: string; diff --git a/packages/metro/src/ModuleGraph/test-helpers.js b/packages/metro/src/ModuleGraph/test-helpers.js index f4782b96..29e17c0f 100644 --- a/packages/metro/src/ModuleGraph/test-helpers.js +++ b/packages/metro/src/ModuleGraph/test-helpers.js @@ -21,4 +21,4 @@ exports.fn = () => { const generateOptions = {concise: true}; exports.codeFromAst = ast => generate(ast, generateOptions).code; -exports.comparableCode = code => code.trim().replace(/\s\s+/g, ' '); +exports.comparableCode = code => code.trim().replace(/\s+/g, ' '); diff --git a/packages/metro/src/ModuleGraph/worker/JsFileWrapping.js b/packages/metro/src/ModuleGraph/worker/JsFileWrapping.js index 2b624b5a..cdb6a542 100644 --- a/packages/metro/src/ModuleGraph/worker/JsFileWrapping.js +++ b/packages/metro/src/ModuleGraph/worker/JsFileWrapping.js @@ -41,6 +41,14 @@ function wrapPolyfill(fileAst: Object): Object { return t.file(t.program([t.expressionStatement(iife)])); } +function wrapJson(source: string): string { + return [ + `__d(function(${MODULE_FACTORY_PARAMETERS.join(', ')}) {`, + ` module.exports = ${source};`, + `});`, + ].join('\n'); +} + function functionFromProgram( program: Object, parameters: Array, @@ -73,8 +81,7 @@ function renameRequires(ast: Object) { } module.exports = { - MODULE_FACTORY_PARAMETERS, - POLYFILL_FACTORY_PARAMETERS, + wrapJson, wrapModule, wrapPolyfill, }; diff --git a/packages/metro/src/ModuleGraph/worker/__tests__/JsFileWrapping-test.js b/packages/metro/src/ModuleGraph/worker/__tests__/JsFileWrapping-test.js index 8bb6b378..ae8fb107 100644 --- a/packages/metro/src/ModuleGraph/worker/__tests__/JsFileWrapping-test.js +++ b/packages/metro/src/ModuleGraph/worker/__tests__/JsFileWrapping-test.js @@ -100,6 +100,41 @@ it('wraps a polyfill correctly', () => { ); }); +it('wraps a JSON file correctly', () => { + const source = JSON.stringify( + { + foo: 'foo', + bar: 'bar', + baz: true, + qux: null, + arr: [1, 2, 3, 4], + }, + null, + 2, + ); + + const wrappedJson = JsFileWrapping.wrapJson(source); + + expect(comparableCode(wrappedJson)).toEqual( + comparableCode( + `__d(function(global, require, module, exports) { + module.exports = { + "foo": "foo", + "bar": "bar", + "baz": true, + "qux": null, + "arr": [ + 1, + 2, + 3, + 4 + ] + }; + });`, + ), + ); +}); + function astFromCode(code) { return babylon.parse(code, {plugins: ['dynamicImport']}); } diff --git a/packages/metro/src/ModuleGraph/worker/__tests__/transform-module-test.js b/packages/metro/src/ModuleGraph/worker/__tests__/transform-module-test.js index 0541daf8..46912f18 100644 --- a/packages/metro/src/ModuleGraph/worker/__tests__/transform-module-test.js +++ b/packages/metro/src/ModuleGraph/worker/__tests__/transform-module-test.js @@ -214,7 +214,7 @@ describe('transforming JS modules:', () => { const {code} = result.details.transformed.default; expect(code.replace(/\s+/g, '')).toEqual( '__d(function(global,require,module,exports){' + - `module.exports=${json}});`, + `module.exports=${json};});`, ); }); diff --git a/packages/metro/src/ModuleGraph/worker/transform-module.js b/packages/metro/src/ModuleGraph/worker/transform-module.js index 78ea3364..9790bd11 100644 --- a/packages/metro/src/ModuleGraph/worker/transform-module.js +++ b/packages/metro/src/ModuleGraph/worker/transform-module.js @@ -125,9 +125,7 @@ function transformModule( function transformJSON(json, options): TransformedSourceFile { const value = JSON.parse(json); const {filename} = options; - const code = `__d(function(${JsFileWrapping.MODULE_FACTORY_PARAMETERS.join( - ', ', - )}) { module.exports = \n${json}\n});`; + const code = JsFileWrapping.wrapJson(json); const moduleData = { code,