Move worker protocol and babelRegisterOnly into their own packages

Summary: Moves the implementation of Buck’s worker protocol into its own package and babelRegisterOnly for better reusability.

Reviewed By: rafeca

Differential Revision: D7666896

fbshipit-source-id: ae297494ced3b8dd1f9d90983a640643d6ce7896
This commit is contained in:
David Aurelio 2018-04-23 03:52:15 -07:00 committed by Facebook Github Bot
parent c398cd99a1
commit bce317701b
11 changed files with 789 additions and 9 deletions

View File

@ -0,0 +1,26 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
'use strict';
/**
* Thrown to indicate the command failed and already output relevant error
* information on the console.
*/
class CommandFailedError extends Error {
constructor() {
super(
'The Buck worker-tool command failed. Diagnostics should have ' +
'been printed on the standard error output.',
);
}
}
module.exports = CommandFailedError;

View File

@ -0,0 +1,433 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @emails oncall+javascript_foundation
*/
'use strict';
jest
.mock('console')
.mock('fs', () => new (require('metro-memory-fs'))())
.mock('temp', () => ({path() {
return '/tmp/repro.args';
}}))
.useRealTimers();
const JSONStream = require('JSONStream');
const buckWorker = require('../worker-tool');
const path = require('path');
const mkdirp = require('mkdirp');
// mocked
const {Console} = require('console');
const fs = require('fs');
const {any, anything} = jasmine;
const UNKNOWN_MESSAGE = 1;
const INVALID_MESSAGE = 2;
describe('Buck worker:', () => {
let commands, inStream, worker, written;
beforeEach(() => {
commands = {};
worker = buckWorker(commands);
inStream = JSONStream.stringify();
inStream.pipe(worker);
written = [];
worker.on('data', chunk => written.push(chunk));
});
describe('handshake:', () => {
it('responds to a correct handshake', () => {
inStream.write(handshake());
return end().then(data =>
expect(data).toEqual([handshake()])
);
});
it('responds to a handshake with a `protocol_version` different from "0"', () => {
inStream.write({
id: 0,
type: 'handshake',
protocol_version: '2',
capabilities: [],
});
return end().then(responses =>
expect(responses).toEqual([{
id: 0,
type: 'error',
exit_code: INVALID_MESSAGE,
}]));
});
it('errors for a second handshake', () => {
inStream.write(handshake());
inStream.write(handshake(1));
return end().then(([, response]) =>
expect(response).toEqual({
id: 1,
type: 'error',
exit_code: UNKNOWN_MESSAGE,
}));
});
});
it('errors for unknown message types', () => {
inStream.write(handshake());
inStream.write({id: 1, type: 'arbitrary'});
return end().then(([, response]) =>
expect(response).toEqual({
id: 1,
type: 'error',
exit_code: UNKNOWN_MESSAGE,
}));
});
describe('commands:', () => {
let createWriteStreamImpl, openedStreams;
function mockFiles(files) {
writeFiles(files, '/');
}
function writeFiles(files, dirPath) {
for (const key in files) {
const entry = files[key];
if (entry == null || typeof entry === 'string') {
fs.writeFileSync(path.join(dirPath, key), entry || '');
} else {
const subDirPath = path.join(dirPath, key);
mkdirp.sync(subDirPath);
writeFiles(entry, subDirPath);
}
}
}
beforeAll(() => {
createWriteStreamImpl = fs.createWriteStream;
fs.createWriteStream = (...args) => {
const writeStream = createWriteStreamImpl(...args);
++openedStreams;
writeStream.on('close', () => (--openedStreams));
return writeStream;
};
});
afterAll(() => {
fs.createWriteStream = createWriteStreamImpl;
});
beforeEach(() => {
fs.reset();
openedStreams = 0;
mockFiles({
'arbitrary': {
'args': '',
'stdout': '',
'stderr': '',
},
// When an error happens, the worker writes a repro file to the
// temporary folder.
'tmp': {},
});
inStream.write(handshake());
});
afterEach(function assertThatAllWriteStreamsWereClosed() {
expect(openedStreams).toBe(0);
});
it('errors if `args_path` cannot be opened', () => {
mockFiles({some: {'args-path': undefined}});
inStream.write(command({id: 5, args_path: '/some/args-path'}));
return end(2).then(([, response]) => {
expect(response).toEqual({
id: 5,
type: 'error',
exit_code: INVALID_MESSAGE,
});
});
});
it('errors if `stdout_path` cannot be opened', () => {
const path = '/does/not/exist';
inStream.write(command({id: 5, stdout_path: path}));
return end(2).then(([, response]) => {
expect(response).toEqual({
id: 5,
type: 'error',
exit_code: INVALID_MESSAGE,
});
});
});
it('errors if `stderr_path` cannot be opened', () => {
const path = '/does/not/exist';
inStream.write(command({id: 5, stderr_path: path}));
return end(2).then(([, response]) => {
expect(response).toEqual({
id: 5,
type: 'error',
exit_code: INVALID_MESSAGE,
});
});
});
it('errors for unspecified commands', () => {
mockFiles({
arbitrary: {
file: '--flag-without-preceding-command',
},
});
inStream.write(command({
id: 1,
args_path: '/arbitrary/file',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 1,
type: 'error',
exit_code: INVALID_MESSAGE,
}));
});
it('errors for empty commands', () => {
mockFiles({
arbitrary: {
file: '',
},
});
inStream.write(command({
id: 2,
args_path: '/arbitrary/file',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 2,
type: 'error',
exit_code: INVALID_MESSAGE,
}));
});
it('errors for unknown commands', () => {
mockFiles({
arbitrary: {
file: 'arbitrary',
},
});
inStream.write(command({
id: 3,
args_path: '/arbitrary/file',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 3,
type: 'error',
exit_code: INVALID_MESSAGE,
}));
});
it('errors if no `args_path` is specified', () => {
inStream.write({
id: 1,
type: 'command',
stdout_path: '/arbitrary',
stderr_path: '/arbitrary',
});
return end().then(([, response]) =>
expect(response)
.toEqual({id: 1, type: 'error', exit_code: INVALID_MESSAGE}));
});
it('errors if no `stdout_path` is specified', () => {
inStream.write({
id: 1,
type: 'command',
args_path: '/arbitrary',
stderr_path: '/arbitrary',
});
return end().then(([, response]) =>
expect(response)
.toEqual({id: 1, type: 'error', exit_code: INVALID_MESSAGE}));
});
it('errors if no `stderr_path` is specified', () => {
inStream.write({
id: 1,
type: 'command',
args_path: '/arbitrary',
stdout_path: '/arbitrary',
});
return end(2).then(([, response]) =>
expect(response)
.toEqual({id: 1, type: 'error', exit_code: INVALID_MESSAGE}));
});
it('passes arguments to an existing command', () => {
commands.transform = jest.fn();
const args = 'foo bar baz\tmore';
mockFiles({
arbitrary: {
file: 'transform ' + args,
},
});
inStream.write(command({
args_path: '/arbitrary/file',
}));
return end(1).then(() =>
expect(commands.transform)
.toBeCalledWith(args.split(/\s+/), null, anything()));
});
it('passes JSON/structured arguments to an existing command', async () => {
commands.transform = jest.fn();
const args = {foo: 'bar', baz: 'glo'};
mockFiles({
arbitrary: {
file: JSON.stringify({...args, command: 'transform'}),
},
});
inStream.write(command({
args_path: '/arbitrary/file',
}));
await end(1);
expect(commands.transform)
.toBeCalledWith([], args, anything());
});
it('passes a console object to the command', () => {
mockFiles({
args: 'transform',
stdio: {},
});
commands.transform = jest.fn();
inStream.write(command({
args_path: '/args',
stdout_path: '/stdio/out',
stderr_path: '/stdio/err',
}));
return end().then(() => {
const streams = last(Console.mock.calls);
expect(streams[0].path).toEqual('/stdio/out');
expect(streams[1].path).toEqual('/stdio/err');
expect(commands.transform)
.toBeCalledWith(anything(), null, any(Console));
});
});
it('responds with success if the command finishes succesfully', () => {
commands.transform = (args, _) => {};
mockFiles({path: {to: {args: 'transform'}}});
inStream.write(command({
id: 123,
args_path: '/path/to/args',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 123,
type: 'result',
exit_code: 0,
}));
});
it('responds with error if the command errors asynchronously', () => {
commands.transform =
jest.fn((args, _, callback) => Promise.reject(new Error('arbitrary')));
mockFiles({path: {to: {args: 'transform'}}});
inStream.write(command({
id: 123,
args_path: '/path/to/args',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 123,
type: 'error',
exit_code: 3,
}));
});
it('responds with error if the command throws synchronously', () => {
commands.transform = (args, _) => {
throw new Error('arbitrary');
};
mockFiles({path: {to: {args: 'transform'}}});
inStream.write(command({
id: 456,
args_path: '/path/to/args',
}));
return end(2).then(([, response]) =>
expect(response).toEqual({
id: 456,
type: 'error',
exit_code: 3,
}));
});
});
function end(afterMessages) {
return new Promise((resolve, reject) => {
worker
.once('error', reject)
.once('end', () => resolve(written.join('')));
if (afterMessages == null || written.length >= afterMessages) {
inStream.end();
} else {
worker.on('data', () => {
if (written.length === afterMessages) {
inStream.end();
}
});
}
}).then(JSON.parse);
}
});
function command(overrides) {
return {
id: 4, // chosen by fair dice roll
type: 'command',
args_path: '/arbitrary/args',
stdout_path: '/arbitrary/stdout',
stderr_path: '/arbitrary/stderr',
...overrides,
};
}
function handshake(id = 0) {
return {
id,
type: 'handshake',
protocol_version: '0',
capabilities: [],
};
}
function last(arrayLike) {
return arrayLike[arrayLike.length - 1];
}

View File

@ -0,0 +1,18 @@
{
"name": "buck-worker-tool",
"version": "0.0.0",
"description": "Implementation of the Buck worker protocol for Node.js.",
"license": "MIT",
"main": "worker-tool.js",
"dependencies": {
"JSONStream": "^1.3.1",
"async": "^2.4.0",
"duplexer": "^0.1.1",
"fbjs": "^0.8.14",
"temp": "^0.8.3"
},
"devDependencies": {
"metro-memory-fs": "*",
"mkdirp": "^0.5.1"
}
}

View File

@ -0,0 +1,277 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
'use strict';
const CommandFailedError = require('./CommandFailedError');
const JSONStream = require('JSONStream');
const duplexer = require('duplexer');
const each = require('async/each');
const fs = require('fs');
const invariant = require('fbjs/lib/invariant');
const path = require('path');
const temp = require('temp');
const {Console} = require('console');
import type {Writable} from 'stream';
export type Command = (
argv: Array<string>,
structuredArgs: mixed,
console: Console,
) => Promise<void> | void;
export type Commands = {[key: string]: Command};
type Message<Type: string, Data> = Data & {
id: number,
type: Type,
};
type HandshakeMessage = Message<
'handshake',
{
protocol_version: '0',
capabilities: [],
},
>;
type CommandMessage = Message<
'command',
{
args_path: string,
stdout_path: string,
stderr_path: string,
},
>;
type UnknownMessage = Message<any, {}>;
type HandshakeReponse = HandshakeMessage;
type CommandResponse = Message<
'result',
{
exit_code: 0,
},
>;
type ErrorResponse = Message<
'error',
{
exit_code: number,
},
>;
type IncomingMessage = HandshakeMessage | CommandMessage | UnknownMessage;
type Response = HandshakeReponse | CommandResponse | ErrorResponse;
type RespondFn = (response: Response) => void;
type JSONReader = {
on: ((
type: 'data',
listener: (message: IncomingMessage) => any,
) => JSONReader) &
((type: 'end') => JSONReader),
removeListener(type: 'data' | 'end', listener: Function): JSONReader,
};
type JSONWriter = {
write(object: Response): void,
end(object?: Response): void,
};
function buckWorker(commands: Commands) {
const reader: JSONReader = JSONStream.parse('*');
const writer: JSONWriter = JSONStream.stringify();
function handleHandshake(message: IncomingMessage) {
const response = handShakeResponse(message);
writer.write(response);
if (response.type === 'handshake') {
reader.removeListener('data', handleHandshake).on('data', handleCommand);
}
}
function handleCommand(message: CommandMessage | UnknownMessage) {
const {id} = message;
if (message.type !== 'command') {
writer.write(unknownMessage(id));
return;
}
/* $FlowFixMe(>=0.44.0 site=react_native_fb) Flow error found while
* deploying v0.44.0. Remove this comment to see the error */
if (!message.args_path || !message.stdout_path || !message.stderr_path) {
writer.write(invalidMessage(id));
return;
}
let responded: boolean = false;
let stdout, stderr;
try {
stdout = fs.createWriteStream(message.stdout_path);
stderr = fs.createWriteStream(message.stderr_path);
} catch (e) {
respond(invalidMessage(id));
return;
}
readArgsAndExecCommand(message, commands, stdout, stderr, respond);
function respond(response: Response) {
// 'used for lazy `.stack` access'
invariant(!responded, `Already responded to message id ${id}.`);
responded = true;
each(
[stdout, stderr].filter(Boolean),
(stream, cb) => stream.end(cb),
error => {
if (error) {
throw error;
}
writer.write(response);
},
);
}
}
/* $FlowFixMe(site=react_native_fb) - Flow now prevents you from calling a
* function with more arguments than it expects. This comment suppresses an
* error that was noticed when we made this change. Delete this comment to
* see the error. */
reader.on('data', handleHandshake).on('end', () => writer.end());
return duplexer(reader, writer);
}
function handShakeResponse(message: HandshakeMessage | UnknownMessage) {
return message.type !== 'handshake'
? unknownMessage(message.id)
: /* $FlowFixMe(>=0.44.0 site=react_native_fb) Flow error found while
* deploying v0.44.0. Remove this comment to see the error */
message.protocol_version !== '0'
? invalidMessage(message.id)
: handshake(message.id);
}
function readArgsAndExecCommand(
message: CommandMessage,
commands: Commands,
stdout: Writable,
stderr: Writable,
respond: RespondFn,
) {
const {id} = message;
fs.readFile(message.args_path, 'utf8', (readError, argsString) => {
if (readError) {
respond(invalidMessage(id));
return;
}
let commandName;
let args = [];
let structuredArgs = null;
// If it starts with a left brace, we assume it's JSON-encoded. This works
// because the non-JSON encoding always starts the string with the
// command name, thus a letter.
if (argsString[0] === '{') {
({command: commandName, ...structuredArgs} = JSON.parse(argsString));
} else {
// FIXME: if there are files names with escaped
// whitespace, this will not work.
[commandName, ...args] = argsString.split(/\s+/);
}
if (commands.hasOwnProperty(commandName)) {
const command = commands[commandName];
const commandSpecificConsole = new Console(stdout, stderr);
execCommand(
command,
commandName,
argsString,
args,
structuredArgs,
commandSpecificConsole,
respond,
id,
);
} else {
respond(invalidMessage(id));
}
});
}
const {JS_WORKER_TOOL_DEBUG_RE} = process.env;
const DEBUG_RE = JS_WORKER_TOOL_DEBUG_RE
? new RegExp(JS_WORKER_TOOL_DEBUG_RE)
: null;
async function execCommand(
command: Command,
commandName: string,
argsString: string,
args: Array<string>,
structuredArgs: mixed,
commandSpecificConsole: Console,
respond: RespondFn,
messageId: number,
) {
let makeResponse = success;
try {
if (shouldDebugCommand(argsString)) {
throw new Error(
`Stopping for debugging. Command '${commandName} ...' matched by the 'JS_WORKER_TOOL_DEBUG_RE' environment variable`,
);
}
await command(args.slice(), structuredArgs, commandSpecificConsole);
} catch (e) {
if (!(e instanceof CommandFailedError)) {
commandSpecificConsole.error(e);
}
makeResponse = commandError;
displayDebugMessage(commandSpecificConsole, argsString);
}
respond(makeResponse(messageId));
}
function shouldDebugCommand(argsString) {
return DEBUG_RE && DEBUG_RE.test(argsString);
}
const ENV_VARS_FOR_REPRO = ['GRAPHQL_SCHEMAS_DIR'];
function displayDebugMessage(commandSpecificConsole, argsString) {
const binPath = path.resolve(process.cwd(), 'js/metro-buck/cli.js');
const reproPath = temp.path({
prefix: 'packager-buck-worker-repro.',
suffix: '.args',
});
fs.writeFileSync(reproPath, argsString);
const nodePath = process.execPath;
const envVars = ENV_VARS_FOR_REPRO.map(
name => `${name}='${(process.env[name] || '').replace("'", "'\\''")}'`,
).join(' ');
commandSpecificConsole.error(
'\nTo reproduce, run:\n' +
` ${envVars} ${nodePath} --preserve-symlinks ${binPath} ${reproPath}\n`,
);
}
const error = (id, exitCode) => ({type: 'error', id, exit_code: exitCode});
const handshake = id => ({
id,
type: 'handshake',
protocol_version: '0',
capabilities: [],
});
const unknownMessage = id => error(id, 1);
const invalidMessage = id => error(id, 2);
const commandError = id => error(id, 3);
const success = id => ({type: 'result', id, exit_code: 0});
module.exports = buckWorker;

View File

@ -8,7 +8,7 @@
*/
'use strict';
require('./setupNodePolyfills');
require('./node-polyfills');
var _only = [];

View File

@ -0,0 +1,19 @@
{
"version": "0.34.1",
"name": "metro-babel-register",
"description": "🚇 babel/register configuration for Metro.",
"main": "babel-register.js",
"repository": {
"type": "git",
"url": "git@github.com:facebook/metro.git"
},
"dependencies": {
"@babel/plugin-proposal-class-properties": "7.0.0-beta.40",
"@babel/plugin-proposal-object-rest-spread": "7.0.0-beta.40",
"@babel/plugin-transform-async-to-generator": "7.0.0-beta.40",
"@babel/plugin-transform-flow-strip-types": "7.0.0-beta.40",
"@babel/plugin-transform-modules-commonjs": "7.0.0-beta.40",
"@babel/register": "7.0.0-beta.40",
"core-js": "^2.2.2"
}
}

View File

@ -61,7 +61,6 @@
"chalk": "^1.1.1",
"concat-stream": "^1.6.0",
"connect": "^3.6.5",
"core-js": "^2.2.2",
"debug": "^2.2.0",
"denodeify": "^1.2.1",
"eventemitter3": "^3.0.0",
@ -79,6 +78,7 @@
"merge-stream": "^1.0.1",
"metro-babel7-plugin-react-transform": "0.34.0",
"metro-babylon7": "0.34.0",
"metro-babel-register": "0.34.1",
"metro-cache": "0.34.0",
"metro-core": "0.34.0",
"metro-minify-uglify": "0.34.0",

View File

@ -29,7 +29,7 @@ const inlineRequiresPlugin6 = require('babel-preset-fbjs/plugins/inline-requires
const makeHMRConfig6 = require('babel-preset-react-native/configs/hmr');
const resolvePlugins6 = require('babel-preset-react-native/lib/resolvePlugins');
// register has side effects so don't include by default (only used in a test)
const getBabelRegisterConfig6 = () => require('./babelRegisterOnly').config;
const getBabelRegisterConfig6 = () => require('metro-babel-register').config;
// load given preset as a babel6 preset
const getPreset6 = (preset: string) =>
// $FlowFixMe TODO t26372934 plugin require
@ -152,7 +152,7 @@ function makeMakeHMRConfig7() {
}
function getPreset7() {
// from: fbsource/xplat/js/node_modules/babel-preset-react-native/configs/main.js
// from: babel-preset-react-native/configs/main.js
/**
* Copyright (c) 2015-present, Facebook, Inc.
*
@ -253,7 +253,7 @@ function getPreset7() {
}
function transformSymbolMember() {
// from: fbsource/xplat/js/node_modules/babel-preset-react-native/transforms/transform-symbol-member.js
// from: babel-preset-react-native/transforms/transform-symbol-member.js
/**
* Copyright (c) 2015-present, Facebook, Inc.
@ -323,7 +323,7 @@ function transformSymbolMember() {
}
function transformDynamicImport() {
// from: fbsource/xplat/js/node_modules/babel-preset-react-native/transforms/transform-dynamic-import.js
// from: babel-preset-react-native/transforms/transform-dynamic-import.js
/**
* Copyright (c) 2015-present, Facebook, Inc.
@ -358,7 +358,7 @@ function transformDynamicImport() {
}
function getBabelRegisterConfig7() {
// from: fbsource/xplat/js/metro/packages/metro/src/babelRegisterOnly.js
// from: metro/packages/metro-babel-register/babel-register.js
// (dont use babel-register anymore, it obsoleted with babel 7)
/**
@ -371,7 +371,7 @@ function getBabelRegisterConfig7() {
*/
'use strict';
require('./setupNodePolyfills');
require('metro-babel-register/node-polyfills');
var _only = [];

View File

@ -9,4 +9,4 @@
'use strict';
global.Promise = require('promise');
require('../packages/metro/src/setupNodePolyfills');
require('../packages/metro-babel-register/node-polyfills');

View File

@ -462,6 +462,13 @@ JSONStream@^1.0.4:
jsonparse "^1.2.0"
through ">=2.2.7 <3"
JSONStream@^1.3.1:
version "1.3.2"
resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea"
dependencies:
jsonparse "^1.2.0"
through ">=2.2.7 <3"
abab@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/abab/-/abab-1.0.3.tgz#b81de5f7274ec4e756d797cd834f303642724e5d"