Symbolicate stack traces off the main process

Summary:
Moves stack trace symbolication to a worker process.

The worker process is spawned laziliy, and is treated as an exclusive resource (requests are queued).

This helps keeping the server process responsive when symbolicating.

Reviewed By: cpojer

Differential Revision: D4602722

fbshipit-source-id: 5da97e53afd9a1ab981c5ba4b02a7d1d869dee71
This commit is contained in:
David Aurelio 2017-02-27 10:44:28 -08:00 committed by Facebook Github Bot
parent cd85f9a5c2
commit d7dbcfc478
10 changed files with 682 additions and 90 deletions

View File

@ -17,6 +17,7 @@
"babel-register": "^6.18.0",
"babylon": "^6.14.1",
"chalk": "^1.1.1",
"concat-stream": "^1.6.0",
"core-js": "^2.2.2",
"debug": "^2.2.0",
"denodeify": "^1.2.1",
@ -38,6 +39,7 @@
"throat": "^3.0.0",
"uglify-js": "^2.6.2",
"worker-farm": "^1.3.1",
"write-file-atomic": "^1.2.0"
"write-file-atomic": "^1.2.0",
"xpipe": "^1.0.5"
}
}

View File

@ -10,11 +10,14 @@
jest.disableAutomock();
jest.setMock('worker-farm', function() { return () => {}; })
.setMock('timers', { setImmediate: (fn) => setTimeout(fn, 0) })
.setMock('uglify-js')
.setMock('crypto')
.setMock('source-map', { SourceMapConsumer: function(fn) {}})
jest.mock('worker-farm', () => () => () => {})
.mock('timers', () => ({ setImmediate: (fn) => setTimeout(fn, 0) }))
.mock('uglify-js')
.mock('crypto')
.mock(
'../symbolicate',
() => ({createWorker: jest.fn().mockReturnValue(jest.fn())}),
)
.mock('../../Bundler')
.mock('../../AssetServer')
.mock('../../lib/declareOpts')
@ -23,14 +26,14 @@ jest.setMock('worker-farm', function() { return () => {}; })
.mock('../../lib/GlobalTransformCache');
describe('processRequest', () => {
let SourceMapConsumer, Bundler, Server, AssetServer, Promise;
let Bundler, Server, AssetServer, Promise, symbolicate;
beforeEach(() => {
jest.resetModules();
SourceMapConsumer = require('source-map').SourceMapConsumer;
Bundler = require('../../Bundler');
Server = require('../');
AssetServer = require('../../AssetServer');
Promise = require('promise');
symbolicate = require('../symbolicate');
});
let server;
@ -69,7 +72,7 @@ describe('processRequest', () => {
Promise.resolve({
getModules: () => [],
getSource: () => 'this is the source',
getSourceMap: () => {},
getSourceMap: () => ({version: 3}),
getSourceMapString: () => 'this is the source map',
getEtag: () => 'this is an etag',
}));
@ -464,60 +467,38 @@ describe('processRequest', () => {
});
describe('/symbolicate endpoint', () => {
let symbolicationWorker;
beforeEach(() => {
symbolicationWorker = symbolicate.createWorker();
symbolicationWorker.mockReset();
});
it('should symbolicate given stack trace', () => {
const body = JSON.stringify({stack: [{
const inputStack = [{
file: 'http://foo.bundle?platform=ios',
lineNumber: 2100,
column: 44,
customPropShouldBeLeftUnchanged: 'foo',
}]});
}];
const outputStack = [{
source: 'foo.js',
line: 21,
column: 4,
}];
const body = JSON.stringify({stack: inputStack});
SourceMapConsumer.prototype.originalPositionFor = jest.fn((frame) => {
expect(frame.line).toEqual(2100);
expect(frame.column).toEqual(44);
return {
source: 'foo.js',
line: 21,
column: 4,
};
expect.assertions(2);
symbolicationWorker.mockImplementation(stack => {
expect(stack).toEqual(inputStack);
return outputStack;
});
return makeRequest(
requestHandler,
'/symbolicate',
{ rawBody: body }
).then(response => {
expect(JSON.parse(response.body)).toEqual({
stack: [{
file: 'foo.js',
lineNumber: 21,
column: 4,
customPropShouldBeLeftUnchanged: 'foo',
}]
});
});
});
it('ignores `/debuggerWorker.js` stack frames', () => {
const body = JSON.stringify({stack: [{
file: 'http://localhost:8081/debuggerWorker.js',
lineNumber: 123,
column: 456,
}]});
return makeRequest(
requestHandler,
'/symbolicate',
{ rawBody: body }
).then(response => {
expect(JSON.parse(response.body)).toEqual({
stack: [{
file: 'http://localhost:8081/debuggerWorker.js',
lineNumber: 123,
column: 456,
}]
});
});
{rawBody: body},
).then(response =>
expect(JSON.parse(response.body)).toEqual({stack: outputStack}));
});
});

View File

@ -15,12 +15,12 @@ const AssetServer = require('../AssetServer');
const getPlatformExtension = require('../node-haste').getPlatformExtension;
const Bundler = require('../Bundler');
const MultipartResponse = require('./MultipartResponse');
const SourceMapConsumer = require('source-map').SourceMapConsumer;
const declareOpts = require('../lib/declareOpts');
const defaults = require('../../defaults');
const mime = require('mime-types');
const path = require('path');
const symbolicate = require('./symbolicate');
const terminal = require('../lib/terminal');
const url = require('url');
@ -34,6 +34,7 @@ import type Bundle from '../Bundler/Bundle';
import type {Reporter} from '../lib/reporting';
import type {GetTransformOptions} from '../Bundler';
import type GlobalTransformCache from '../lib/GlobalTransformCache';
import type {SourceMap, Symbolicate} from './symbolicate';
const {
createActionStartEntry,
@ -201,6 +202,7 @@ class Server {
_debouncedFileChangeHandler: (filePath: string) => mixed;
_hmrFileChangeListener: (type: string, filePath: string) => mixed;
_reporter: Reporter;
_symbolicateInWorker: Symbolicate;
constructor(options: Options) {
this._opts = {
@ -281,6 +283,8 @@ class Server {
}
this._informChangeWatchers();
}, 50);
this._symbolicateInWorker = symbolicate.createWorker();
}
end(): mixed {
@ -802,45 +806,27 @@ class Server {
// In case of multiple bundles / HMR, some stack frames can have
// different URLs from others
const urlIndexes = {};
const uniqueUrls = [];
const urls = new Set();
stack.forEach(frame => {
const sourceUrl = frame.file;
// Skip `/debuggerWorker.js` which drives remote debugging because it
// does not need to symbolication.
// Skip anything except http(s), because there is no support for that yet
if (!urlIndexes.hasOwnProperty(sourceUrl) &&
if (!urls.has(sourceUrl) &&
!sourceUrl.endsWith('/debuggerWorker.js') &&
sourceUrl.startsWith('http')) {
urlIndexes[sourceUrl] = uniqueUrls.length;
uniqueUrls.push(sourceUrl);
urls.add(sourceUrl);
}
});
const sourceMaps = uniqueUrls.map(
sourceUrl => this._sourceMapForURL(sourceUrl)
);
return Promise.all(sourceMaps).then(consumers => {
return stack.map(frame => {
const sourceUrl = frame.file;
if (!urlIndexes.hasOwnProperty(sourceUrl)) {
return frame;
}
const idx = urlIndexes[sourceUrl];
const consumer = consumers[idx];
const original = consumer.originalPositionFor({
line: frame.lineNumber,
column: frame.column,
});
if (!original) {
return frame;
}
return Object.assign({}, frame, {
file: original.source,
lineNumber: original.line,
column: original.column,
});
});
const mapPromises =
Array.from(urls.values()).map(this._sourceMapForURL, this);
debug('Getting source maps for symbolication');
return Promise.all(mapPromises).then(maps => {
debug('Sending stacks and maps to symbolication worker');
const urlsToMaps = zip(urls.values(), maps);
return this._symbolicateInWorker(stack, urlsToMaps);
});
}).then(
stack => {
@ -858,16 +844,13 @@ class Server {
);
}
_sourceMapForURL(reqUrl: string): Promise<SourceMapConsumer> {
_sourceMapForURL(reqUrl: string): Promise<SourceMap> {
const options = this._getOptionsFromUrl(reqUrl);
const building = this._useCachedOrUpdateOrCreateBundle(options);
return building.then(p => {
const sourceMap = p.getSourceMap({
minify: options.minify,
dev: options.dev,
});
return new SourceMapConsumer(sourceMap);
});
return building.then(p => p.getSourceMap({
minify: options.minify,
dev: options.dev,
}));
}
_handleError(res: ServerResponse, bundleID: string, error: {
@ -990,4 +973,16 @@ function contentsEqual(array: Array<mixed>, set: Set<mixed>): boolean {
return array.length === set.size && array.every(set.has, set);
}
function* zip<X, Y>(xs: Iterable<X>, ys: Iterable<Y>): Iterable<[X, Y]> {
//$FlowIssue #9324959
const ysIter: Iterator<Y> = ys[Symbol.iterator]();
for (const x of xs) {
const y = ysIter.next();
if (y.done) {
return;
}
yield [x, y.value];
}
}
module.exports = Server;

View File

@ -0,0 +1,106 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.disableAutomock();
jest.mock('child_process');
jest.mock('net');
const EventEmitter = require('events');
const {Readable} = require('stream');
const {createWorker} = require('../');
let childProcess, socketResponse, socket, worker;
beforeEach(() => {
childProcess = Object.assign(new EventEmitter(), {send: jest.fn()});
require('child_process').fork.mockReturnValueOnce(childProcess);
setupCommunication();
socketResponse = '{"error": "no fake socket response set"}';
socket = Object.assign(new Readable(), {
_read() {
this.push(socketResponse);
this.push(null);
},
end: jest.fn(),
setEncoding: jest.fn(),
});
require('net').createConnection.mockImplementation(() => socket);
worker = createWorker();
});
it('sends a socket path to the child process', () => {
socketResponse = '{}';
return worker([], fakeSourceMaps())
.then(() => expect(childProcess.send).toBeCalledWith(expect.any(String)));
});
it('fails if the child process emits an error', () => {
const error = new Error('Expected error');
childProcess.send.mockImplementation(() =>
childProcess.emit('error', error));
expect.assertions(1);
return worker([], fakeSourceMaps())
.catch(e => expect(e).toBe(error));
});
it('fails if the socket connection emits an error', () => {
const error = new Error('Expected error');
socket._read = () => socket.emit('error', error);
expect.assertions(1);
return worker([], fakeSourceMaps())
.catch(e => expect(e).toBe(error));
});
it('sends the passed in stack and maps over the socket', () => {
socketResponse = '{}';
const stack = ['the', 'stack'];
return worker(stack, fakeSourceMaps())
.then(() =>
expect(socket.end).toBeCalledWith(JSON.stringify({
maps: Array.from(fakeSourceMaps()),
stack,
})));
});
it('resolves to the `result` property of the message returned over the socket', () => {
socketResponse = '{"result": {"the": "result"}}';
return worker([], fakeSourceMaps())
.then(response => expect(response).toEqual({the: 'result'}));
});
it('rejects with the `error` property of the message returned over the socket', () => {
socketResponse = '{"error": "the error message"}';
expect.assertions(1);
return worker([], fakeSourceMaps())
.catch(error => expect(error).toEqual(new Error('the error message')));
});
it('rejects if the socket response cannot be parsed as JSON', () => {
socketResponse = '{';
expect.assertions(1);
return worker([], fakeSourceMaps())
.catch(error => expect(error).toBeInstanceOf(SyntaxError));
});
function setupCommunication() {
childProcess.send.mockImplementation(() =>
process.nextTick(() => childProcess.emit('message')));
}
function* fakeSourceMaps() {
yield [1, {}];
yield [2, {}];
}

View File

@ -0,0 +1,157 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.disableAutomock();
const {LazyPromise, LockingPromise} = require('../util');
describe('Lazy Promise', () => {
let factory;
const value = {};
beforeEach(() => {
factory = jest.fn();
factory.mockReturnValue(Promise.resolve(value));
});
it('does not run the factory by default', () => {
new LazyPromise(factory); // eslint-disable-line no-new
expect(factory).not.toBeCalled();
});
it('calling `.then()` returns a promise', () => {
expect(new LazyPromise(factory).then()).toBeInstanceOf(Promise);
});
it('does not invoke the factory twice', () => {
const p = new LazyPromise(factory);
p.then(x => x);
p.then(x => x);
expect(factory).toHaveBeenCalledTimes(1);
});
describe('value and error propagation', () => {
it('resolves to the value provided by the factory', () => {
expect.assertions(1);
return new LazyPromise(factory)
.then(v => expect(v).toBe(value));
});
it('passes through errors if not handled', () => {
const error = new Error('Unhandled');
factory.mockReturnValue(Promise.reject(error));
expect.assertions(1);
return new LazyPromise(factory)
.then()
.catch(e => expect(e).toBe(error));
});
it('uses rejection handlers passed to `then()`', () => {
const error = new Error('Must be handled');
factory.mockReturnValue(Promise.reject(error));
expect.assertions(1);
return new LazyPromise(factory)
.then(() => {}, e => expect(e).toBe(error));
});
it('uses rejection handlers passed to `catch()`', () => {
const error = new Error('Must be handled');
factory.mockReturnValue(Promise.reject(error));
expect.assertions(1);
return new LazyPromise(factory)
.catch(e => expect(e).toBe(error));
});
});
});
describe('Locking Promise', () => {
it('resolves to the value of the passed-in promise', () => {
const value = {};
expect.assertions(1);
return new LockingPromise(Promise.resolve(value))
.then(v => expect(v).toBe(value));
});
it('passes through rejections', () => {
const error = new Error('Rejection');
expect.assertions(1);
return new LockingPromise(Promise.reject(error))
.then()
.catch(e => expect(e).toBe(error));
});
it('uses rejection handlers passed to `then()`', () => {
const error = new Error('Must be handled');
expect.assertions(1);
return new LockingPromise(Promise.reject(error))
.then(x => x, e => expect(e).toBe(error));
});
it('uses rejection handlers passed to `catch()`', () => {
const error = new Error('Must be handled');
expect.assertions(1);
return new LockingPromise(Promise.reject(error))
.catch(e => expect(e).toBe(error));
});
describe('locking', () => {
const value = Symbol;
let locking;
beforeEach(() => {
locking = new LockingPromise(Promise.resolve(value));
});
it('only allows one handler to access the promise value', () => {
const deferred = defer();
const secondHandler = jest.fn();
locking.then(() => deferred.promise);
locking.then(secondHandler);
return Promise.resolve() // wait for the next tick
.then(() => expect(secondHandler).not.toBeCalled());
});
it('allows waiting handlers to access the value after the current handler resolves', () => {
let counter = 0;
const deferred = defer();
const x = locking.then(v => {
const result = [++counter, v];
return deferred.promise.then(() => result);
});
const y = locking.then(v => [++counter, v]);
const z = locking.then(v => [++counter, v]);
deferred.resolve();
return Promise.all([x, y, z])
.then(([first, second, third]) => {
expect(first).toEqual([1, value]);
expect(second).toEqual([2, value]);
expect(third).toEqual([3, value]);
});
});
});
});
function defer() {
let resolve;
const promise = new Promise(res => { resolve = res; });
return {promise, resolve};
}

View File

@ -0,0 +1,111 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
jest.disableAutomock();
const SourceMapGenerator = require('../../../Bundler/source-map/Generator');
const {symbolicate} = require('../worker');
let connection;
beforeEach(() => {
connection = {end: jest.fn()};
});
it('symbolicates stack frames', () => {
const mappings = [
{
from: {file: 'bundle1.js', lineNumber: 1, column: 2},
to: {file: 'apples.js', lineNumber: 12, column: 34},
},
{
from: {file: 'bundle2.js', lineNumber: 3, column: 4},
to: {file: 'bananas.js', lineNumber: 56, column: 78},
},
{
from: {file: 'bundle1.js', lineNumber: 5, column: 6},
to: {file: 'clementines.js', lineNumber: 90, column: 12},
},
];
const stack = mappings.map(m => m.to);
const maps =
Object.entries(groupBy(mappings, m => m.from.file))
.map(([file, ms]) => [file, sourceMap(file, ms)]);
return symbolicate(connection, makeData(stack, maps))
.then(() =>
expect(connection.end).toBeCalledWith(
JSON.stringify({result: mappings.map(m => m.to)})
)
);
});
it('ignores stack frames without corresponding map', () => {
const frame = {
file: 'arbitrary.js',
lineNumber: 123,
column: 456,
};
return symbolicate(connection, makeData([frame], [['other.js', emptyMap()]]))
.then(() =>
expect(connection.end).toBeCalledWith(
JSON.stringify({result: [frame]})
)
);
});
it('ignores `/debuggerWorker.js` stack frames', () => {
const frame = {
file: 'http://localhost:8081/debuggerWorker.js',
lineNumber: 123,
column: 456,
};
return symbolicate(connection, makeData([frame]))
.then(() =>
expect(connection.end).toBeCalledWith(
JSON.stringify({result: [frame]})
)
);
});
function makeData(stack, maps = []) {
return JSON.stringify({maps, stack});
}
function sourceMap(file, mappings) {
const g = new SourceMapGenerator();
g.startFile(file, null);
mappings.forEach(({from, to}) =>
g.addSourceMapping(to.lineNumber, to.column, from.lineNumber, from.column));
return g.toMap();
}
function groupBy(xs, key) {
const grouped = {};
xs.forEach(x => {
const k = key(x);
if (k in grouped) {
grouped[k].push(x);
} else {
grouped[k] = [x];
}
});
return grouped;
}
function emptyMap() {
return {
version: 3,
sources: [],
mappings: '',
};
}

View File

@ -0,0 +1 @@
{"main": "./symbolicate.js"}

View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
const concat = require('concat-stream');
const debug = require('debug')('RNP:Symbolication');
const net = require('net');
const temp = require('temp');
const xpipe = require('xpipe');
const {LazyPromise, LockingPromise} = require('./util');
const {fork} = require('child_process');
export type {SourceMap as SourceMap};
import type {MixedSourceMap as SourceMap} from '../../lib/SourceMap';
export type Stack = Array<{file: string, lineNumber: number, column: number}>;
export type Symbolicate =
(Stack, Iterable<[string, SourceMap]>) => Promise<Stack>;
const affixes = {prefix: 'metro-bundler-symbolicate', suffix: '.sock'};
const childPath = require.resolve('./worker');
exports.createWorker = (): Symbolicate => {
const socket = xpipe.eq(temp.path(affixes));
const child = new LockingPromise(new LazyPromise(() => startupChild(socket)));
return (stack, sourceMaps) =>
child
.then(() => connectAndSendJob(socket, message(stack, sourceMaps)))
.then(JSON.parse)
.then(response =>
'error' in response
? Promise.reject(new Error(response.error))
: response.result
);
};
function startupChild(socket) {
const child = fork(childPath);
return new Promise((resolve, reject) => {
child
.once('error', reject)
.once('message', () => {
child.removeAllListeners();
resolve(child);
});
child.send(socket);
});
}
function connectAndSendJob(socket, data) {
const job = new Promise((resolve, reject) => {
debug('Connecting to worker');
const connection = net.createConnection(socket);
connection.setEncoding('utf8');
connection.on('error', reject);
connection.pipe(concat(resolve));
debug('Sending data to worker');
connection.end(data);
});
job.then(() => debug('Received response from worker'));
return job;
}
function message(stack, sourceMaps) {
return JSON.stringify({maps: Array.from(sourceMaps), stack});
}

View File

@ -0,0 +1,86 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* @flow
*/
'use strict';
type PromiseLike<R> = {
catch<U>(onReject?: (error: any) => ?Promise<U> | U): Promise<U>,
then<U>(
fulfilled?: R => Promise<U> | U,
rejected?: (error: any) => Promise<U> | U,
): Promise<U>,
};
/**
* A promise-like object that only creates the underlying value lazily
* when requested.
*/
exports.LazyPromise = class LazyPromise<T> {
_promise: PromiseLike<T>;
constructor(factory: () => PromiseLike<T>) {
//$FlowIssue #16209141
Object.defineProperty(this, '_promise', {
configurable: true,
enumerable: true,
get: () => (this._promise = factory()),
set: value => Object.defineProperty(this, '_promise', {value}),
});
}
then<U>(
fulfilled?: (value: T) => Promise<U> | U,
rejected?: (error: any) => Promise<U> | U
): Promise<U> {
return this._promise.then(fulfilled, rejected);
}
catch<U>(
rejected?: (error: any) => ?Promise<U> | U
): Promise<U> {
return this._promise.catch(rejected);
}
};
/**
* A promise-like object that allows only one `.then()` handler to access
* the wrapped value simultaneously. Can be used to lock resources that do
* asynchronous work.
*/
exports.LockingPromise = class LockingPromise<T> {
_gate: PromiseLike<any>
_promise: PromiseLike<T>
constructor(promise: PromiseLike<T>) {
this._gate = this._promise = promise;
}
then<U>(
fulfilled?: (value: T) => Promise<U> | U,
rejected?: (error: any) => Promise<U> | U
): Promise<U> {
const whenUnlocked = () => {
const promise = this._promise.then(fulfilled, rejected);
this._gate = promise.then(empty); // avoid retaining the result of promise
return promise;
};
return this._gate.then(whenUnlocked, whenUnlocked);
}
catch<U>(
rejected?: (error: any) => ?Promise<U> | U
): Promise<U> {
return this._promise.catch(rejected);
}
};
function empty() {}

View File

@ -0,0 +1,77 @@
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
'use strict';
// RUNS UNTRANSFORMED IN NODE >= v4
// NO FANCY FEATURES, E.G. DESTRUCTURING, PLEASE!
const SourceMapConsumer = require('source-map').SourceMapConsumer;
const concat = require('concat-stream');
const net = require('net');
process.once('message', socket => {
net.createServer({allowHalfOpen: true}, connection => {
connection.setEncoding('utf8');
connection.pipe(concat(data =>
symbolicate(connection, data)
.catch(console.error) // log the error as a last resort
));
}).listen(socket, () => process.send(null));
});
function symbolicate(connection, data) {
return Promise.resolve(data)
.then(JSON.parse)
.then(symbolicateStack)
.then(JSON.stringify)
.catch(makeErrorMessage)
.then(message => connection.end(message));
}
function symbolicateStack(data) {
const consumers = new Map(data.maps.map(mapToConsumer));
return {
result: data.stack.map(frame => mapFrame(frame, consumers)),
};
}
function mapFrame(frame, consumers) {
const sourceUrl = frame.file;
const consumer = consumers.get(sourceUrl);
if (consumer == null) {
return frame;
}
const original = consumer.originalPositionFor({
line: frame.lineNumber,
column: frame.column,
});
if (!original) {
return frame;
}
return Object.assign({}, frame, {
file: original.source,
lineNumber: original.line,
column: original.column,
});
}
function makeErrorMessage(error) {
return JSON.stringify({
error: String(error && error.message || error),
});
}
function mapToConsumer(tuple) {
tuple[1] = new SourceMapConsumer(tuple[1]);
return tuple;
}
// for testing
exports.symbolicate = symbolicate;