Optimise e2e tests retries
Summary:Implemented smarter retries for e2e tests E2e tests consist of two expensive and flaky steps: npm installation and tests execution. Our CI is configured to retry a command 3 times before terminating the test. In current setup if one of the steps fail the whole test is restarted. This change adds ad-hoc ability to retry flaky bits of e2e script independently. This will make tests fail faster when code gets broken while increasing the chances to succeed in case of random false errors. Closes https://github.com/facebook/react-native/pull/7184 Differential Revision: D3218927 fb-gh-sync-id: 9be8343484bb28aa3601b651db70fc55aa840e61 fbshipit-source-id: 9be8343484bb28aa3601b651db70fc55aa840e61
This commit is contained in:
Konstantin Raev
Facebook Github Bot 9
parent
a2ee5bd307
commit
a8d2079022
|
|
@ -15,7 +15,7 @@ install:
|
|||
|
||||
script:
|
||||
- if [[ "$TEST_TYPE" = objc ]]; then travis_retry ./scripts/objc-test.sh; fi
|
||||
- if [[ "$TEST_TYPE" = e2e-objc ]]; then travis_retry node ./scripts/run-ci-e2e-tests.js --ios --js; fi
|
||||
- if [[ "$TEST_TYPE" = e2e-objc ]]; then node ./scripts/run-ci-e2e-tests.js --ios --js --retries 3; fi
|
||||
- if [[ "$TEST_TYPE" = js ]]; then npm run flow check; fi
|
||||
- if [[ "$TEST_TYPE" = js ]]; then npm test -- --maxWorkers=1; fi
|
||||
|
||||
|
|
|
|||
|
|
@ -76,9 +76,7 @@ test:
|
|||
- ./gradlew :ReactAndroid:assembleDebugAndroidTest
|
||||
|
||||
# Android e2e test
|
||||
# TODO t10955118, temporary until master is fixed
|
||||
#- source scripts/circle-ci-android-setup.sh && retry3 node ./scripts/run-ci-e2e-tests.js --android --js
|
||||
- node ./scripts/run-ci-e2e-tests.js --android --js
|
||||
- node ./scripts/run-ci-e2e-tests.js --android --js --retries 3
|
||||
|
||||
# testing docs generation is not broken
|
||||
- cd website && node ./server/generate.js
|
||||
|
|
|
|||
|
|
@ -16,92 +16,114 @@
|
|||
* --android - to test only android application end to end
|
||||
* --js - to test that JS in the application is compilable
|
||||
* --skip-cli-install - to skip react-native-cli global installation (for local debugging)
|
||||
* --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1
|
||||
*/
|
||||
/*eslint-disable no-undef */
|
||||
require('shelljs/global');
|
||||
var spawn = require('child_process').spawn;
|
||||
|
||||
const spawn = require('child_process').spawn;
|
||||
const argv = require('yargs').argv;
|
||||
const path = require('path');
|
||||
|
||||
const SCRIPTS = __dirname;
|
||||
const ROOT = path.normalize(path.join(__dirname, '..'));
|
||||
|
||||
const TEMP=exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
|
||||
const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
|
||||
// To make sure we actually installed the local version
|
||||
// of react-native, we will create a temp file inside the template
|
||||
// and check that it exists after `react-native init
|
||||
const MARKER_IOS = exec(`mktemp ${ROOT}/local-cli/generator-ios/templates/app/XXXXXXXX`).stdout.trim();
|
||||
const MARKER_ANDROID = exec(`mktemp ${ROOT}/local-cli/generator-android/templates/src/XXXXXXXX`).stdout.trim();
|
||||
|
||||
const numberOfRetries = argv.retries || 1;
|
||||
let SERVER_PID;
|
||||
let APPIUM_PID;
|
||||
let exitCode;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
function cleanup(errorCode) {
|
||||
if (errorCode !== 0) {
|
||||
cat(`${TEMP}/server.log`);
|
||||
cat(`/usr/local/Cellar/watchman/3.1/var/run/watchman/${process.env.USER}.log`);
|
||||
/**
|
||||
* Try executing a function n times recursively.
|
||||
* Return 0 the first time it succeeds
|
||||
* Return code of the last failed commands if not more retries left
|
||||
* @funcToRetry - function that gets retried
|
||||
* @retriesLeft - number of retries to execute funcToRetry
|
||||
* @onEveryError - func to execute if funcToRetry returns non 0
|
||||
*/
|
||||
function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
|
||||
const exitCode = funcToRetry();
|
||||
if (exitCode === 0) {
|
||||
return exitCode;
|
||||
} else {
|
||||
if (onEveryError) {
|
||||
onEveryError();
|
||||
}
|
||||
rm(MARKER_IOS);
|
||||
rm(MARKER_ANDROID);
|
||||
|
||||
if(SERVER_PID) {
|
||||
echo(`Killing packager ${SERVER_PID}`);
|
||||
exec(`kill -9 ${SERVER_PID}`);
|
||||
// this is quite drastic but packager starts a daemon that we can't kill by killing the parent process
|
||||
// it will be fixed in April (quote David Aurelio), so until then we will kill the zombie by the port number
|
||||
exec("lsof -i tcp:8081 | awk 'NR!=1 {print $2}' | xargs kill");
|
||||
retriesLeft--;
|
||||
echo(`Command failed, ${retriesLeft} retries left`);
|
||||
if (retriesLeft === 0) {
|
||||
return exitCode;
|
||||
} else {
|
||||
return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
|
||||
}
|
||||
if(APPIUM_PID) {
|
||||
echo(`Killing appium ${APPIUM_PID}`);
|
||||
exec(`kill -9 ${APPIUM_PID}`);
|
||||
}
|
||||
return errorCode;
|
||||
}
|
||||
|
||||
// install CLI
|
||||
cd('react-native-cli');
|
||||
exec('npm pack');
|
||||
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
|
||||
cd('..');
|
||||
try {
|
||||
// install CLI
|
||||
cd('react-native-cli');
|
||||
exec('npm pack');
|
||||
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
|
||||
cd('..');
|
||||
|
||||
// can skip cli install for non sudo mode
|
||||
if(args.indexOf('--skip-cli-install') === -1) {
|
||||
if(exec(`npm install -g ${CLI_PACKAGE}`).code) {
|
||||
// can skip cli install for non sudo mode
|
||||
if (!argv['skip-cli-install']) {
|
||||
if (exec(`npm install -g ${CLI_PACKAGE}`).code) {
|
||||
echo('Could not install react-native-cli globally, please run in su mode');
|
||||
echo('Or with --skip-cli-install to skip this step');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.indexOf('--android') !== -1) {
|
||||
if (argv['android']) {
|
||||
if (exec('./gradlew :ReactAndroid:installArchives -Pjobs=1 -Dorg.gradle.jvmargs="-Xmx512m -XX:+HeapDumpOnOutOfMemoryError"').code) {
|
||||
echo('Failed to compile Android binaries');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (exec('npm pack').code) {
|
||||
if (exec('npm pack').code) {
|
||||
echo('Failed to pack react-native');
|
||||
exit(cleanup(1));
|
||||
}
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
|
||||
// test begins
|
||||
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
|
||||
cd(TEMP);
|
||||
if (exec(`react-native init EndToEndTest --version ${PACKAGE}`).code) {
|
||||
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
|
||||
cd(TEMP);
|
||||
if (tryExecNTimes(
|
||||
() => {
|
||||
exec('sleep 10s');
|
||||
return exec(`react-native init EndToEndTest --version ${PACKAGE}`).code;
|
||||
},
|
||||
numberOfRetries,
|
||||
() => rm('-rf', 'EndToEndTest'))) {
|
||||
echo('Failed to execute react-native init');
|
||||
echo('Most common reason is npm registry connectivity, try again');
|
||||
exit(cleanup(1));
|
||||
}
|
||||
cd('EndToEndTest');
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
|
||||
if (args.indexOf('--android') !== -1) {
|
||||
cd('EndToEndTest');
|
||||
|
||||
if (argv['android']) {
|
||||
echo('Running an Android e2e test');
|
||||
echo('Installing e2e framework');
|
||||
if(exec('npm install --save-dev appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1', {silent: true}).code) {
|
||||
if (tryExecNTimes(
|
||||
() => exec('npm install --save-dev appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1', { silent: true }).code,
|
||||
numberOfRetries)) {
|
||||
echo('Failed to install appium');
|
||||
exit(cleanup(1));
|
||||
echo('Most common reason is npm registry connectivity, try again');
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
cp(`${SCRIPTS}/android-e2e-test.js`, 'android-e2e-test.js');
|
||||
cd('android');
|
||||
|
|
@ -110,7 +132,8 @@ if (args.indexOf('--android') !== -1) {
|
|||
// Make sure we installed local version of react-native
|
||||
if (!test('-e', path.basename(MARKER_ANDROID))) {
|
||||
echo('Android marker was not found, react native init command failed?');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
cd('..');
|
||||
exec('keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"');
|
||||
|
|
@ -122,7 +145,8 @@ if (args.indexOf('--android') !== -1) {
|
|||
echo('Building app');
|
||||
if (exec('buck build android/app').code) {
|
||||
echo('could not execute Buck build, is it installed and in PATH?');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
let packagerEnv = Object.create(process.env);
|
||||
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
|
||||
|
|
@ -135,18 +159,27 @@ if (args.indexOf('--android') !== -1) {
|
|||
// wait a bit to allow packager to startup
|
||||
exec('sleep 5s');
|
||||
echo('Executing android e2e test');
|
||||
if(exec('node node_modules/.bin/_mocha android-e2e-test.js').code) {
|
||||
exit(cleanup(1));
|
||||
if (tryExecNTimes(
|
||||
() => {
|
||||
exec('sleep 10s');
|
||||
return exec('node node_modules/.bin/_mocha android-e2e-test.js').code;
|
||||
},
|
||||
numberOfRetries)) {
|
||||
echo('Failed to run Android e2e tests');
|
||||
echo('Most likely the code is broken');
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.indexOf('--ios') !== -1) {
|
||||
if (argv['ios']) {
|
||||
echo('Running an iOS app');
|
||||
cd('ios');
|
||||
// Make sure we installed local version of react-native
|
||||
if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) {
|
||||
echo('iOS marker was not found, `react-native init` command failed?');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
|
||||
let packagerEnv = Object.create(process.env);
|
||||
|
|
@ -162,27 +195,57 @@ if (args.indexOf('--ios') !== -1) {
|
|||
exec('response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.ios.bundle?platform=ios&dev=true)');
|
||||
echo(`Starting packager server, ${SERVER_PID}`);
|
||||
echo('Executing ios e2e test');
|
||||
if (exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code) {
|
||||
exit(cleanup(1));
|
||||
if (tryExecNTimes(
|
||||
() => {
|
||||
exec('sleep 10s');
|
||||
return exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code;
|
||||
},
|
||||
numberOfRetries)) {
|
||||
echo('Failed to run iOS e2e tests');
|
||||
echo('Most likely the code is broken');
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
cd('..');
|
||||
}
|
||||
}
|
||||
|
||||
if (args.indexOf('--js') !== -1) {
|
||||
if (argv['js']) {
|
||||
// Check the packager produces a bundle (doesn't throw an error)
|
||||
if (exec('react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
|
||||
echo('Could not build android package');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
if (exec('react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
|
||||
echo('Could not build ios package');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
if (exec(`${ROOT}/node_modules/.bin/flow check`).code) {
|
||||
echo('Flow check does not pass');
|
||||
exit(cleanup(1));
|
||||
exitCode = 1;
|
||||
throw Error(exitCode);
|
||||
}
|
||||
}
|
||||
exitCode = 0;
|
||||
|
||||
} finally {
|
||||
cd(ROOT);
|
||||
rm(MARKER_IOS);
|
||||
rm(MARKER_ANDROID);
|
||||
|
||||
if (SERVER_PID) {
|
||||
echo(`Killing packager ${SERVER_PID}`);
|
||||
exec(`kill -9 ${SERVER_PID}`);
|
||||
// this is quite drastic but packager starts a daemon that we can't kill by killing the parent process
|
||||
// it will be fixed in April (quote David Aurelio), so until then we will kill the zombie by the port number
|
||||
exec("lsof -i tcp:8081 | awk 'NR!=1 {print $2}' | xargs kill");
|
||||
}
|
||||
if (APPIUM_PID) {
|
||||
echo(`Killing appium ${APPIUM_PID}`);
|
||||
exec(`kill -9 ${APPIUM_PID}`);
|
||||
}
|
||||
}
|
||||
exit(exitCode);
|
||||
|
||||
exit(cleanup(0));
|
||||
/*eslint-enable no-undef */
|
||||
|
|
|
|||
Loading…
Reference in New Issue