From b2032b05be549469173dc1d70b3bab4022c48b35 Mon Sep 17 00:00:00 2001 From: Freddy Date: Mon, 10 Sep 2018 16:44:07 +0100 Subject: [PATCH] Add script and makefile goal to help debug flaky tests --- GNUmakefile | 3 + build-support/docker/Test-Flake.dockerfile | 13 ++ build-support/docker/flake.sh | 29 ++++ build-support/scripts/test-flake.sh | 169 +++++++++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 build-support/docker/Test-Flake.dockerfile create mode 100644 build-support/docker/flake.sh create mode 100755 build-support/scripts/test-flake.sh diff --git a/GNUmakefile b/GNUmakefile index 3286ced169..547309dfb9 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -194,6 +194,9 @@ test-ci: other-consul dev-build vet test-install-deps fi \ fi +test-flake: other-consul vet test-install-deps + @$(SHELL) $(CURDIR)/build-support/scripts/test-flake.sh --pkg "$(FLAKE_PKG)" --test "$(FLAKE_TEST)" --cpus "$(FLAKE_CPUS)" --n "$(FLAKE_N)" + other-consul: @echo "--> Checking for other consul instances" @if ps -ef | grep 'consul agent' | grep -v grep ; then \ diff --git a/build-support/docker/Test-Flake.dockerfile b/build-support/docker/Test-Flake.dockerfile new file mode 100644 index 0000000000..11d8896b6b --- /dev/null +++ b/build-support/docker/Test-Flake.dockerfile @@ -0,0 +1,13 @@ +FROM travisci/ci-garnet:packer-1512502276-986baf0 + +ENV GOLANG_VERSION 1.10.3 + +RUN mkdir -p /home/travis/go && chown -R travis /home/travis/go + +ENV GOPATH /home/travis/go + +USER travis + +COPY flake.sh /usr/local/bin/flake.sh + +ENTRYPOINT [ "bash", "flake.sh" ] \ No newline at end of file diff --git a/build-support/docker/flake.sh b/build-support/docker/flake.sh new file mode 100644 index 0000000000..0a885015d8 --- /dev/null +++ b/build-support/docker/flake.sh @@ -0,0 +1,29 @@ +#!/bin/bash +LOG_FILE="test.log" + +cd $GOPATH/$APP + +export PATH=$GOPATH/$APP/bin:$GOPATH/bin:$PATH + +if ! [[ $(ls -l | grep 'GNUmakefile\|README.md\|LICENSE') ]] ; then + echo "App source not present in cwd. Exiting..." + exit 1 +fi + +mv $TEST_BINARY $TEST_PKG/$TEST_BINARY +cd $TEST_PKG + +for ((i=0; i < $ITERATIONS; i++)) ; do + echo "$(date +"%F %T") - ($((i+1))/$ITERATIONS)" + + ./$TEST_BINARY -test.run "$TEST" -test.parallel 4 -test.timeout 8m -test.v &> $LOG_FILE + echo $? > exit-code + + grep -A10 'panic: ' $LOG_FILE || true + awk '/^[^[:space:]]/ {do_print=0} /--- SKIP/ {do_print=1} do_print==1 {print}' $LOG_FILE + awk '/^[^[:space:]]/ {do_print=0} /--- FAIL/ {do_print=1} do_print==1 {print}' $LOG_FILE + + if [ $(cat exit-code) != "0" ] ; then + exit 1; + fi +done \ No newline at end of file diff --git a/build-support/scripts/test-flake.sh b/build-support/scripts/test-flake.sh new file mode 100755 index 0000000000..38e5879edd --- /dev/null +++ b/build-support/scripts/test-flake.sh @@ -0,0 +1,169 @@ +#!/bin/bash +pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null +SCRIPT_DIR=$(pwd) +pushd ../.. > /dev/null +SOURCE_DIR=$(pwd) +pushd build-support/docker > /dev/null +IMG_DIR=$(pwd) +popd > /dev/null + +source "${SCRIPT_DIR}/functions.sh" + +IMAGE="travis-img-v0.13" +CONTAINER="travis-cnt" +GOOS="linux" +GOARCH="amd64" +TEST_BINARY="flake.test" + +function usage { +cat <<-EOF +Usage: test-flake [] + +Description: + + test-flake surfaces flakiness in tests by constraining CPU resources. + + Single or package-wide tests are run for multiple iterations with a configurable + amount of CPU resources. + + 0.15 CPUs and 30 iterations are configured as sane defaults. + + See Docker docs for more info on tuning 'cpus' param: + https://docs.docker.com/config/containers/resource_constraints/#cpu + +Options: + + --pkg="" Target package + --test="" Target test (requires pkg flag) + --cpus=0.15 Amount of CPU resources for container + --n=30 Number of times to run tests + +Examples: + + ./test-flake.sh --pkg connect/proxy + ./test-flake.sh --pkg connect/proxy --cpus 0.20 + ./test-flake.sh --pkg connect/proxy --test Listener + ./test-flake.sh --pkg connect/proxy --test TestUpstreamListener + ./test-flake.sh --pkg connect/proxy --test TestUpstreamListener -n 100 +EOF +} + +function build_repro_env { + # Arguments: + # $1 - pkg, Target package + # $2 - test, Target tests + # $3 - cpus, Amount of CPU resources for container + # $4 - n, Number of times to run tests + + APP=$(pwd | awk '{n=split($0, a, "/"); print a[n]}') + + status_stage -e "App:\t\t$APP" + status_stage -e "Package:\t$1" + status_stage -e "Test:\t\t$2" + status_stage -e "CPUs:\t\t$3" + status_stage -e "Iterations:\t$4" + echo + + status_stage "----> Cleaning up old containers..." + if docker ps -a | grep $CONTAINER ; then + docker rm $(docker ps -a | grep $CONTAINER | awk '{print $1;}') + fi + + status_stage '----> Rebuilding image...' + (cd $IMG_DIR && docker build -q -t $IMAGE --no-cache -f Test-Flake.dockerfile .) + + status_stage "--> Building app binary..." + env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/$APP + + status_stage "-> Building test binary..." + env GOOS=$GOOS GOARCH=$GOARCH go test -c "./$1" -o $TEST_BINARY + + status_stage "> Running container..." + status_stage + + docker run \ + --rm \ + --name $CONTAINER \ + --cpus="$3" \ + -v $SOURCE_DIR:/home/travis/go/$APP \ + -e TEST_BINARY="$TEST_BINARY" \ + -e TEST_PKG="$1" \ + -e TEST="$2" \ + -e ITERATIONS="$4" \ + -e APP="$APP" \ + $IMAGE +} + +function err_usage { + err "$1" + err "" + err "$(usage)" +} + +function main { + declare pkg="" + declare test="" + declare cpus="" + declare n="" + + + while test $# -gt 0 + do + case "$1" in + -h | --help ) + usage + return 0 + ;; + + --pkg ) + if test -z "$2" + then + err_usage "ERROR: option pkg requires an argument" + return 1 + fi + + pkg="$2" + shift 2 + ;; + + --test ) + test="$2" + shift 2 + ;; + + --cpus ) + if test -n "$2" + then + cpus="$2" + else + cpus="0.15" + fi + + shift 2 + ;; + + --n ) + if test -n "$2" + then + n="$2" + else + n="30" + fi + + shift 2 + ;; + + * ) + err_usage "ERROR: Unknown argument: '$1'" + return 1 + ;; + esac + done + + build_repro_env "${pkg}" "${test}" "${cpus}" "${n}" || return 1 + + return 0 +} + +main "$@" +exit $? \ No newline at end of file