Container Testing Updates

Summary:
ericvicenti - Here are necessary updates for the the testing container workflow.

normanjoyner

Temporarily disabled E2E tests on the Containership Jenkins pipeline because they seemed to be sporadically failing and slow down the run rate of CI tests.

* Run all stages and parallel branches in testing suite even if one
fails (just mark the overall job as failed)
* Added update to docker tag name to convert spaces to hyphens to cover
edge case where the project name may have spaces
* Updated buck to more recent version in the base image
* Remove duplicate module provider from `.flowconfig`
* Correctly exit with status code from failed instrumentation tests
* Add JSCHeaders to prepare for testing on stage branches
* Fix bug in filtering of failed instrumentation tests
* Turn down retry count to 1 for E2E tests and temporarily disable
* Add retry3 count to the apk install from buck
* Updated base image to install android SDKs through grepping since the
IDs update frequ
Closes https://github.com/facebook/react-native/pull/13032

Differential Revision: D4799015

Pulled By: ericvicenti

fbshipit-source-id: bbac9303e8ca4fe8be0e8b230a2f863c71c3366c

ContainerShip/Dockerfile.android

@@ -33,7 +33,7 @@ ADD build.gradle /app/build.gradle
 ADD react.gradle /app/react.gradle
 
 # run gradle downloads
-RUN ./gradlew :ReactAndroid:downloadBoost :ReactAndroid:downloadDoubleConversion :ReactAndroid:downloadFolly :ReactAndroid:downloadGlog
+RUN ./gradlew :ReactAndroid:downloadBoost :ReactAndroid:downloadDoubleConversion :ReactAndroid:downloadFolly :ReactAndroid:downloadGlog :ReactAndroid:downloadJSCHeaders
 
 # compile native libs with Gradle script, we need bridge for unit and integration tests
 RUN ./gradlew :ReactAndroid:packageReactNdkLibsForBuck -Pjobs=1 -Pcom.android.build.threadPoolSize=1

ContainerShip/Dockerfile.android-base

@@ -2,7 +2,7 @@ FROM library/ubuntu:16.04
 
 # set default build arguments
 ARG ANDROID_VERSION=25.2.3
-ARG BUCK_VERSION=2016.11.11.01
+ARG BUCK_VERSION=f3452a6a7ab15a60e94c962e686293acbe677473
 ARG NDK_VERSION=10e
 ARG NODE_VERSION=6.2.0
 ARG WATCHMAN_VERSION=4.7.0
@@ -30,7 +30,7 @@ RUN n $NODE_VERSION
 # download buck
 RUN git clone https://github.com/facebook/buck.git /opt/buck
 WORKDIR /opt/buck
-RUN git checkout v$BUCK_VERSION
+RUN git checkout $BUCK_VERSION
 
 # build buck
 RUN ant
@@ -62,16 +62,33 @@ RUN unzip ndk.zip
 # cleanup NDK
 RUN rm ndk.zip
 
-# add android SDK tools
-# 2 - Android SDK Platform-tools, revision 25.0.3
-# 12 - Android SDK Build-tools, revision 23.0.1
-# 35 - SDK Platform Android 6.0, API 23, revision 3
-# 39 - SDK Platform Android 4.4.2, API 19, revision 4
-# 102 - ARM EABI v7a System Image, Android API 19, revision 5
-# 103 - Intel x86 Atom System Image, Android API 19, revision 5
-# 131 - Google APIs, Android API 23, revision 1
-# 166 - Android Support Repository, revision 41
-RUN echo "y" | android update sdk -u -a -t 2,12,35,39,102,103,131,166
+# Add android SDK tools
+
+# Android SDK Platform-tools, revision 25.0.4
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Android SDK Platform-tools, revision 25.0.4" | awk '{ print $1 }' | sed 's/.$//')
+
+# Android SDK Build-tools, revision 23.0.1
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Android SDK Build-tools, revision 23.0.1" | awk '{ print $1 }' | sed 's/.$//')
+
+# SDK Platform Android 6.0, API 23, revision 3
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "SDK Platform Android 6.0, API 23" | awk '{ print $1 }' | sed 's/.$//')
+
+# SDK Platform Android 4.4.2, API 19, revision 4
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "SDK Platform Android 4.4.2, API 19, revision 4" | awk '{ print $1 }' | sed 's/.$//')
+
+# ARM EABI v7a System Image, Android API 19, revision 5
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "ARM EABI v7a System Image, Android API 19, revision 5" | awk '{ print $1 }' | sed 's/.$//')
+
+# Intel x86 Atom System Image, Android API 19, revision 5
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Intel x86 Atom System Image, Android API 19, revision 5" | awk '{ print $1 }' | sed 's/.$//')
+
+# Google APIs, Android API 23, revision 1
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Google APIs, Android API 23, revision 1" | awk '{ print $1 }' | sed 's/.$//')
+
+# Android Support Repository, revision 45
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Android Support Repository, revision 45" | awk '{ print $1 }' | sed 's/.$//')
+
+# Link adb executable
 RUN ln -s /opt/android/platform-tools/adb /usr/bin/adb
 
 # clean up unnecessary directories

ContainerShip/scripts/run-android-ci-instrumentation-tests.js

@@ -28,6 +28,12 @@ const child_process = require('child_process');
 const fs = require('fs');
 const path = require('path');
 
+// Flaky tests ignored on Circle CI. They still run internally at fb.
+const ignoredTests = [
+  'ReactScrollViewTestCase',
+  'ReactHorizontalScrollViewTestCase'
+];
+
 const colors = {
     GREEN: '\x1b[32m',
     RED: '\x1b[31m',
@@ -53,6 +59,8 @@ let testClasses = fs.readdirSync(path.resolve(process.cwd(), test_opts.PATH))
         return file.endsWith('.java');
     }).map((clazz) => {
         return path.basename(clazz, '.java');
+    }).filter(className => {
+        return ignoredTests.indexOf(className) === -1;
     }).map((clazz) => {
         return test_opts.PACKAGE + '.' + clazz;
     }).filter((clazz) => {
@@ -112,7 +120,7 @@ return async.mapSeries(testClasses, (clazz, callback) => {
     print_test_suite_results(results);
 
     const failures = results.filter((test) => {
-        test.status === 'failure';
+        return test.status === 'failure';
     });
 
     return failures.length === 0 ? process.exit(0) : process.exit(1);

ContainerShip/scripts/run-android-docker-instrumentation-tests.sh

@@ -28,7 +28,8 @@ watchman shutdown-server
 node local-cli/cli.js bundle --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
 
 # build test APK
-buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
+source ./scripts/circle-ci-android-setup.sh && NO_BUCKD=1 retry3 buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
 
 # run installed apk with tests
 node ./ContainerShip/scripts/run-android-ci-instrumentation-tests.js $*
+exit $?

ContainerShip/scripts/run-ci-e2e-tests.sh

@@ -11,7 +11,7 @@ RUN_CLI_INSTALL=1
 RUN_IOS=0
 RUN_JS=0
 
-RETRY_COUNT=${RETRY_COUNT:-3}
+RETRY_COUNT=${RETRY_COUNT:-1}
 AVD_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)
 
 ANDROID_NPM_DEPS="appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1"

Jenkinsfile

@@ -99,14 +99,14 @@ def runStages() {
                 buildInfo.scm.sha = githubInfo.sha
                 buildInfo.scm.tag = githubInfo.tag
                 buildInfo.scm.isPR = githubInfo.isPR
-                buildInfo.image.tag = "${buildInfo.scm.sha}-${env.BUILD_TAG.replace("/", "-").replace("%2F", "-")}"
+                buildInfo.image.tag = "${buildInfo.scm.sha}-${env.BUILD_TAG.replace(" ", "-").replace("/", "-").replace("%2F", "-")}"
 
                 jsTag = "${buildInfo.image.tag}"
                 androidTag = "${buildInfo.image.tag}"
                 jsImageName = "${buildInfo.image.name}-js:${jsTag}"
                 androidImageName = "${buildInfo.image.name}-android:${androidTag}"
 
-                parallelInstrumentationTests = getParallelInstrumentationTests('./ReactAndroid/src/androidTest/java/com/facebook/react/tests', 3, androidImageName)
+                parallelInstrumentationTests = getParallelInstrumentationTests('./ReactAndroid/src/androidTest/java/com/facebook/react/tests', 1, androidImageName)
 
                 parallel(
                     'javascript build': {
@@ -120,36 +120,53 @@ def runStages() {
             }
 
             stage('Tests JS') {
-                parallel(
-                    'javascript flow': {
-                        runCmdOnDockerImage(jsImageName, 'yarn run flow -- check', '--rm')
-                    },
-                    'javascript tests': {
-                        runCmdOnDockerImage(jsImageName, 'yarn test --maxWorkers=4', '--rm')
-                    },
-                    'documentation tests': {
-                        runCmdOnDockerImage(jsImageName, 'cd website && yarn test', '--rm')
-                    },
-                    'documentation generation': {
-                        runCmdOnDockerImage(jsImageName, 'cd website && node ./server/generate.js', '--rm')
-                    }
-                )
+                try {
+                    parallel(
+                        'javascript flow': {
+                            runCmdOnDockerImage(jsImageName, 'yarn run flow -- check', '--rm')
+                        },
+                        'javascript tests': {
+                            runCmdOnDockerImage(jsImageName, 'yarn test --maxWorkers=4', '--rm')
+                        },
+                        'documentation tests': {
+                            runCmdOnDockerImage(jsImageName, 'cd website && yarn test', '--rm')
+                        },
+                        'documentation generation': {
+                            runCmdOnDockerImage(jsImageName, 'cd website && node ./server/generate.js', '--rm')
+                        }
+                    )
+                } catch(e) {
+                    currentBuild.result = "FAILED"
+                    echo "Test JS Stage Error: ${e}"
+                }
             }
 
             stage('Tests Android') {
-                parallel(
-                    'android unit tests': {
-                        runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-android-docker-unit-tests.sh', '--privileged --rm')
-                    },
-                    'android e2e tests': {
-                        runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-ci-e2e-tests.sh --android --js', '--rm')
-                    }
-                )
+                try {
+                    parallel(
+                        'android unit tests': {
+                            runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-android-docker-unit-tests.sh', '--privileged --rm')
+                        },
+                        'android e2e tests': {
+                            // temporarily disable e2e tests, they have a high transient failure rate
+                            // runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-ci-e2e-tests.sh --android --js', '--rm')
+                            echo "Android E2E tests have been temporarily disabled"
+                        }
+                    )
+                } catch(e) {
+                    currentBuild.result = "FAILED"
+                    echo "Tests Android Stage Error: ${e}"
+                }
             }
 
             stage('Tests Android Instrumentation') {
                 // run all tests in parallel
-                parallel(parallelInstrumentationTests)
+                try {
+                    parallel(parallelInstrumentationTests)
+                } catch(e) {
+                    currentBuild.result = "FAILED"
+                    echo "Tests Android Instrumentation Stage Error: ${e}"
+                }
             }
 
             stage('Cleanup') {