status-im
/
react-native
mirror of https://github.com/status-im/react-native.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Optimise e2e tests retries 

Summary:Implemented smarter retries for e2e tests

E2e tests consist of two expensive and flaky steps: npm installation and tests execution.
Our CI is configured to retry a command 3 times before terminating the test.
In current setup if one of the steps fail the whole test is restarted.

This change adds ad-hoc ability to retry flaky bits of e2e script independently.
This will make tests fail faster when code gets broken while increasing the chances to succeed in case of random false errors.
Closes https://github.com/facebook/react-native/pull/7184

Differential Revision: D3218927

fb-gh-sync-id: 9be8343484bb28aa3601b651db70fc55aa840e61
fbshipit-source-id: 9be8343484bb28aa3601b651db70fc55aa840e61
This commit is contained in:
Konstantin Raev 2016-04-25 07:15:25 -07:00 committed by Facebook Github Bot 9
parent a2ee5bd307
commit a8d2079022
3 changed files with 202 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -15,7 +15,7 @@ install:
script:
- if [[ "$TEST_TYPE" = objc ]]; then travis_retry ./scripts/objc-test.sh; fi
- if [[ "$TEST_TYPE" = e2e-objc ]]; then travis_retry node ./scripts/run-ci-e2e-tests.js --ios --js; fi
- if [[ "$TEST_TYPE" = e2e-objc ]]; then node ./scripts/run-ci-e2e-tests.js --ios --js --retries 3; fi
- if [[ "$TEST_TYPE" = js ]]; then npm run flow check; fi
- if [[ "$TEST_TYPE" = js ]]; then npm test -- --maxWorkers=1; fi

4
circle.yml
View File

@ -76,9 +76,7 @@ test:
- ./gradlew :ReactAndroid:assembleDebugAndroidTest
# Android e2e test
# TODO t10955118, temporary until master is fixed
#- source scripts/circle-ci-android-setup.sh && retry3 node ./scripts/run-ci-e2e-tests.js --android --js
- node ./scripts/run-ci-e2e-tests.js --android --js
- node ./scripts/run-ci-e2e-tests.js --android --js --retries 3
# testing docs generation is not broken
- cd website && node ./server/generate.js

337
scripts/run-ci-e2e-tests.js
View File

@ -16,173 +16,236 @@
* --android - to test only android application end to end
* --js - to test that JS in the application is compilable
* --skip-cli-install - to skip react-native-cli global installation (for local debugging)
* --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1
*/
/*eslint-disable no-undef */
require('shelljs/global');
var spawn = require('child_process').spawn;
const spawn = require('child_process').spawn;
const argv = require('yargs').argv;
const path = require('path');
const SCRIPTS = __dirname;
const ROOT = path.normalize(path.join(__dirname, '..'));
const TEMP=exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
// To make sure we actually installed the local version
// of react-native, we will create a temp file inside the template
// and check that it exists after `react-native init
const MARKER_IOS = exec(`mktemp ${ROOT}/local-cli/generator-ios/templates/app/XXXXXXXX`).stdout.trim();
const MARKER_ANDROID = exec(`mktemp ${ROOT}/local-cli/generator-android/templates/src/XXXXXXXX`).stdout.trim();
const numberOfRetries = argv.retries || 1;
let SERVER_PID;
let APPIUM_PID;
let exitCode;
const args = process.argv.slice(2);
function cleanup(errorCode) {
if (errorCode !== 0) {
cat(`${TEMP}/server.log`);
cat(`/usr/local/Cellar/watchman/3.1/var/run/watchman/${process.env.USER}.log`);
/**
* Try executing a function n times recursively.
* Return 0 the first time it succeeds
* Return code of the last failed commands if not more retries left
* @funcToRetry - function that gets retried
* @retriesLeft - number of retries to execute funcToRetry
* @onEveryError - func to execute if funcToRetry returns non 0
*/
function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
const exitCode = funcToRetry();
if (exitCode === 0) {
return exitCode;
} else {
if (onEveryError) {
onEveryError();
}
retriesLeft--;
echo(`Command failed, ${retriesLeft} retries left`);
if (retriesLeft === 0) {
return exitCode;
} else {
return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
}
}
}
try {
// install CLI
cd('react-native-cli');
exec('npm pack');
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
cd('..');
// can skip cli install for non sudo mode
if (!argv['skip-cli-install']) {
if (exec(`npm install -g ${CLI_PACKAGE}`).code) {
echo('Could not install react-native-cli globally, please run in su mode');
echo('Or with --skip-cli-install to skip this step');
exitCode = 1;
throw Error(exitCode);
}
}
if (argv['android']) {
if (exec('./gradlew :ReactAndroid:installArchives -Pjobs=1 -Dorg.gradle.jvmargs="-Xmx512m -XX:+HeapDumpOnOutOfMemoryError"').code) {
echo('Failed to compile Android binaries');
exitCode = 1;
throw Error(exitCode);
}
}
if (exec('npm pack').code) {
echo('Failed to pack react-native');
exitCode = 1;
throw Error(exitCode);
}
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
cd(TEMP);
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec(`react-native init EndToEndTest --version ${PACKAGE}`).code;
},
numberOfRetries,
() => rm('-rf', 'EndToEndTest'))) {
echo('Failed to execute react-native init');
echo('Most common reason is npm registry connectivity, try again');
exitCode = 1;
throw Error(exitCode);
}
cd('EndToEndTest');
if (argv['android']) {
echo('Running an Android e2e test');
echo('Installing e2e framework');
if (tryExecNTimes(
() => exec('npm install --save-dev appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1', { silent: true }).code,
numberOfRetries)) {
echo('Failed to install appium');
echo('Most common reason is npm registry connectivity, try again');
exitCode = 1;
throw Error(exitCode);
}
cp(`${SCRIPTS}/android-e2e-test.js`, 'android-e2e-test.js');
cd('android');
echo('Downloading Maven deps');
exec('./gradlew :app:copyDownloadableDepsToLibs');
// Make sure we installed local version of react-native
if (!test('-e', path.basename(MARKER_ANDROID))) {
echo('Android marker was not found, react native init command failed?');
exitCode = 1;
throw Error(exitCode);
}
cd('..');
exec('keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"');
echo(`Starting packager server, ${SERVER_PID}`);
const appiumProcess = spawn('node', ['./node_modules/.bin/appium']);
APPIUM_PID = appiumProcess.pid;
echo(`Starting appium server, ${APPIUM_PID}`);
echo('Building app');
if (exec('buck build android/app').code) {
echo('could not execute Buck build, is it installed and in PATH?');
exitCode = 1;
throw Error(exitCode);
}
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
const packagerProcess = spawn('npm', ['start'], {
// stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
// wait a bit to allow packager to startup
exec('sleep 5s');
echo('Executing android e2e test');
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec('node node_modules/.bin/_mocha android-e2e-test.js').code;
},
numberOfRetries)) {
echo('Failed to run Android e2e tests');
echo('Most likely the code is broken');
exitCode = 1;
throw Error(exitCode);
}
}
if (argv['ios']) {
echo('Running an iOS app');
cd('ios');
// Make sure we installed local version of react-native
if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) {
echo('iOS marker was not found, `react-native init` command failed?');
exitCode = 1;
throw Error(exitCode);
}
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
const packagerProcess = spawn('npm', ['start', '--', '--non-persistent'],
{
stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
exec('sleep 15s');
// prepare cache to reduce chances of possible red screen "Can't fibd variable __fbBatchedBridge..."
exec('response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.ios.bundle?platform=ios&dev=true)');
echo(`Starting packager server, ${SERVER_PID}`);
echo('Executing ios e2e test');
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code;
},
numberOfRetries)) {
echo('Failed to run iOS e2e tests');
echo('Most likely the code is broken');
exitCode = 1;
throw Error(exitCode);
}
cd('..');
}
if (argv['js']) {
// Check the packager produces a bundle (doesn't throw an error)
if (exec('react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
echo('Could not build android package');
exitCode = 1;
throw Error(exitCode);
}
if (exec('react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
echo('Could not build ios package');
exitCode = 1;
throw Error(exitCode);
}
if (exec(`${ROOT}/node_modules/.bin/flow check`).code) {
echo('Flow check does not pass');
exitCode = 1;
throw Error(exitCode);
}
}
exitCode = 0;
} finally {
cd(ROOT);
rm(MARKER_IOS);
rm(MARKER_ANDROID);
if(SERVER_PID) {
if (SERVER_PID) {
echo(`Killing packager ${SERVER_PID}`);
exec(`kill -9 ${SERVER_PID}`);
// this is quite drastic but packager starts a daemon that we can't kill by killing the parent process
// it will be fixed in April (quote David Aurelio), so until then we will kill the zombie by the port number
exec("lsof -i tcp:8081 | awk 'NR!=1 {print $2}' | xargs kill");
}
if(APPIUM_PID) {
if (APPIUM_PID) {
echo(`Killing appium ${APPIUM_PID}`);
exec(`kill -9 ${APPIUM_PID}`);
}
return errorCode;
}
exit(exitCode);
// install CLI
cd('react-native-cli');
exec('npm pack');
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
cd('..');
// can skip cli install for non sudo mode
if(args.indexOf('--skip-cli-install') === -1) {
if(exec(`npm install -g ${CLI_PACKAGE}`).code) {
echo('Could not install react-native-cli globally, please run in su mode');
echo('Or with --skip-cli-install to skip this step');
exit(cleanup(1));
}
}
if (args.indexOf('--android') !== -1) {
if (exec('./gradlew :ReactAndroid:installArchives -Pjobs=1 -Dorg.gradle.jvmargs="-Xmx512m -XX:+HeapDumpOnOutOfMemoryError"').code) {
echo('Failed to compile Android binaries');
exit(cleanup(1));
}
}
if (exec('npm pack').code) {
echo('Failed to pack react-native');
exit(cleanup(1));
}
// test begins
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
cd(TEMP);
if (exec(`react-native init EndToEndTest --version ${PACKAGE}`).code) {
echo('Failed to execute react-native init');
echo('Most common reason is npm registry connectivity, try again');
exit(cleanup(1));
}
cd('EndToEndTest');
if (args.indexOf('--android') !== -1) {
echo('Running an Android e2e test');
echo('Installing e2e framework');
if(exec('npm install --save-dev appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1', {silent: true}).code) {
echo('Failed to install appium');
exit(cleanup(1));
}
cp(`${SCRIPTS}/android-e2e-test.js`, 'android-e2e-test.js');
cd('android');
echo('Downloading Maven deps');
exec('./gradlew :app:copyDownloadableDepsToLibs');
// Make sure we installed local version of react-native
if (!test('-e', path.basename(MARKER_ANDROID))) {
echo('Android marker was not found, react native init command failed?');
exit(cleanup(1));
}
cd('..');
exec('keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"');
echo(`Starting packager server, ${SERVER_PID}`);
const appiumProcess = spawn('node', ['./node_modules/.bin/appium']);
APPIUM_PID = appiumProcess.pid;
echo(`Starting appium server, ${APPIUM_PID}`);
echo('Building app');
if (exec('buck build android/app').code) {
echo('could not execute Buck build, is it installed and in PATH?');
exit(cleanup(1));
}
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
const packagerProcess = spawn('npm', ['start'], {
// stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
// wait a bit to allow packager to startup
exec('sleep 5s');
echo('Executing android e2e test');
if(exec('node node_modules/.bin/_mocha android-e2e-test.js').code) {
exit(cleanup(1));
}
}
if (args.indexOf('--ios') !== -1) {
echo('Running an iOS app');
cd('ios');
// Make sure we installed local version of react-native
if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) {
echo('iOS marker was not found, `react-native init` command failed?');
exit(cleanup(1));
}
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
const packagerProcess = spawn('npm', ['start', '--', '--non-persistent'],
{
stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
exec('sleep 15s');
// prepare cache to reduce chances of possible red screen "Can't fibd variable __fbBatchedBridge..."
exec('response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.ios.bundle?platform=ios&dev=true)');
echo(`Starting packager server, ${SERVER_PID}`);
echo('Executing ios e2e test');
if (exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code) {
exit(cleanup(1));
}
cd('..');
}
if (args.indexOf('--js') !== -1) {
// Check the packager produces a bundle (doesn't throw an error)
if (exec('react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
echo('Could not build android package');
exit(cleanup(1));
}
if (exec('react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
echo('Could not build ios package');
exit(cleanup(1));
}
if (exec(`${ROOT}/node_modules/.bin/flow check`).code) {
echo('Flow check does not pass');
exit(cleanup(1));
}
}
exit(cleanup(0));
/*eslint-enable no-undef */