Added stability to OSS instrumentation tests

Summary:
This change makes all instrumentation tests to be executed in sequence in independent retriable processes.

With a new test being open sourced recently our CI stability degraded.
This PR should bring back stability because tests won't affect each other and will have shorter lifetime
Closes https://github.com/facebook/react-native/pull/7353

Differential Revision: D3259081

fb-gh-sync-id: 48ccdb5dbd561d416526497ff474378db9ca3c60
fbshipit-source-id: 48ccdb5dbd561d416526497ff474378db9ca3c60

ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactPickerTestCase.java

@@ -6,7 +6,7 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
  
-package com.facebook.react;
+package com.facebook.react.tests;
 
 import java.util.ArrayList;
 import java.util.List;

circle.yml

@@ -69,7 +69,7 @@ test:
     # build test APK
     - buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
     # run installed apk with tests
-    - source scripts/circle-ci-android-setup.sh && retry3 ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests
+    - node ./scripts/run-ci-android-instrumentation-tests.js --retries 3 --path ./ReactAndroid/src/androidTest/java/com/facebook/react/tests --package com.facebook.react.tests
 
     - ./gradlew :ReactAndroid:testDebugUnitTest
     # Deprecated: these tests are executed using Buck above, while we support Gradle we just make sure the test code compiles

scripts/run-android-instrumentation-tests.sh

@@ -1,12 +1,16 @@
 #!/bin/bash
 
+# Python script to run instrumentation tests, copied from https://github.com/circleci/circle-dummy-android
+# Example: ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests com.facebook.react.tests.ReactPickerTestCase
+#
 export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/tools:$PATH"
 
 # clear the logs
 adb logcat -c
 
 # run tests and check output
-python - $1 << END
+python - $1 $2 << END
+
 import re
 import subprocess as sp
 import sys
@@ -14,8 +18,13 @@ import threading
 import time
 
 done = False
-test_app = sys.argv[1]
 
+test_app = sys.argv[1]
+test_class = None
+
+if len(sys.argv) > 2:
+  test_class = sys.argv[2]
+  
 def update():
   # prevent CircleCI from killing the process for inactivity
   while not done:
@@ -28,8 +37,12 @@ t.start()
 
 def run():
   sp.Popen(['adb', 'wait-for-device']).communicate()
-  p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner' % test_app,
-               shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
+  if (test_class != None):
+    p = sp.Popen('adb shell am instrument -w -e class %s %s/android.support.test.runner.AndroidJUnitRunner' 
+      % (test_class, test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
+  else :
+    p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner' 
+      % (test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
   return p.communicate()
 
 success = re.compile(r'OK \(\d+ tests\)')

scripts/run-ci-android-instrumentation-tests.js

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+/**
+ * This script runs instrumentation tests one by one with retries
+ * Instrumentation tests tend to be flaky, so rerunning them individually increases
+ * chances for success and reduces total average execution time.
+ *
+ * We assume that all instrumentation tests are flat in one folder
+ * Available arguments:
+ * --path - path to all .java files with tests
+ * --package - com.facebook.react.tests
+ * --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1
+ */
+/*eslint-disable no-undef */
+require('shelljs/global');
+
+const argv = require('yargs').argv;
+const numberOfRetries = argv.retries || 1;
+const tryExecNTimes = require('./try-n-times');
+const path = require('path');
+
+// ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactHorizontalScrollViewTestCase.java
+const testClasses = ls(`${argv.path}/*.java`)
+.map(javaFile => {
+  // ReactHorizontalScrollViewTestCase
+  return path.basename(javaFile, '.java');
+}).map(className => {
+  // com.facebook.react.tests.ReactHorizontalScrollViewTestCase
+  return argv.package + '.' + className;
+});
+
+let exitCode = 0;
+testClasses.forEach((testClass) => {
+  if (tryExecNTimes(
+    () => {
+      exec('sleep 5s');
+      return exec(`./scripts/run-android-instrumentation-tests.sh ${argv.package} ${testClass}`).code;
+    },
+    numberOfRetries)) {
+      echo(`${testClass} failed ${numberOfRetries} times`);
+      exitCode = 1;
+  }
+});
+
+exit(exitCode);
+
+/*eslint-enable no-undef */

scripts/run-ci-e2e-tests.js

@@ -27,6 +27,7 @@ const path = require('path');
 
 const SCRIPTS = __dirname;
 const ROOT = path.normalize(path.join(__dirname, '..'));
+const tryExecNTimes = require('./try-n-times');
 
 const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
 // To make sure we actually installed the local version
@@ -39,33 +40,6 @@ let SERVER_PID;
 let APPIUM_PID;
 let exitCode;
 
-
-/**
- * Try executing a function n times recursively.
- * Return 0 the first time it succeeds
- * Return code of the last failed commands if not more retries left
- * @funcToRetry - function that gets retried
- * @retriesLeft - number of retries to execute funcToRetry
- * @onEveryError - func to execute if funcToRetry returns non 0 
- */
-function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
-  const exitCode = funcToRetry();
-  if (exitCode === 0) {
-    return exitCode;
-  } else {
-    if (onEveryError) {
-      onEveryError();
-    }
-    retriesLeft--;
-    echo(`Command failed, ${retriesLeft} retries left`);
-    if (retriesLeft === 0) {
-      return exitCode;
-    } else {
-      return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
-    }
-  }
-}
-
 try {
   // install CLI
   cd('react-native-cli');

scripts/try-n-times.js

@@ -0,0 +1,27 @@
+/**
+ * Try executing a function n times recursively.
+ * Return 0 the first time it succeeds
+ * Return code of the last failed commands if not more retries left
+ * @funcToRetry - function that gets retried
+ * @retriesLeft - number of retries to execute funcToRetry
+ * @onEveryError - func to execute if funcToRetry returns non 0 
+ */
+function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
+  const exitCode = funcToRetry();
+  if (exitCode === 0) {
+    return exitCode;
+  } else {
+    if (onEveryError) {
+      onEveryError();
+    }
+    retriesLeft--;
+    echo(`Command failed, ${retriesLeft} retries left`);
+    if (retriesLeft === 0) {
+      return exitCode;
+    } else {
+      return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
+    }
+  }
+}
+
+module.exports = tryExecNTimes;