From d46f2429ec77da3453472c17a63edfbf6a8524d7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 21 Oct 2021 16:42:00 -0400 Subject: [PATCH] build-support: remove test-flake machinery This machinery was not used, and does not appear to be maintained. In practice we really don't need anything to detect flaky tests. Our CI system identifies flaky tests at https://app.circleci.com/insights/github/hashicorp/consul/workflows/go-tests/tests?branch=main Mostly what we need is a way to reproduce flakes, which can be done directly with the Go CLI, using the -race, -count, and (new in Go 1.17) -shuffle flags. --- GNUmakefile | 5 +- build-support/docker/Test-Flake.dockerfile | 13 -- build-support/docker/flake.sh | 29 ---- build-support/scripts/test-flake.sh | 169 --------------------- 4 files changed, 1 insertion(+), 215 deletions(-) delete mode 100644 build-support/docker/Test-Flake.dockerfile delete mode 100644 build-support/docker/flake.sh delete mode 100755 build-support/scripts/test-flake.sh diff --git a/GNUmakefile b/GNUmakefile index f7200067d4..90a439f1f0 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -231,9 +231,6 @@ test-internal: test-race: $(MAKE) GOTEST_FLAGS=-race -test-flake: other-consul lint - @$(SHELL) $(CURDIR)/build-support/scripts/test-flake.sh --pkg "$(FLAKE_PKG)" --test "$(FLAKE_TEST)" --cpus "$(FLAKE_CPUS)" --n "$(FLAKE_N)" - test-docker: linux go-build-image @# -ti run in the foreground showing stdout @# --rm removes the container once its finished running @@ -372,6 +369,6 @@ envoy-regen: @find "command/connect/envoy/testdata" -name '*.golden' -delete @go test -tags '$(GOTAGS)' ./command/connect/envoy -update -.PHONY: all bin dev dist cov test test-flake test-internal cover lint ui static-assets tools +.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools .PHONY: docker-images go-build-image ui-build-image static-assets-docker consul-docker ui-docker .PHONY: version proto test-envoy-integ diff --git a/build-support/docker/Test-Flake.dockerfile b/build-support/docker/Test-Flake.dockerfile deleted file mode 100644 index 42d17af0eb..0000000000 --- a/build-support/docker/Test-Flake.dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM travisci/ci-garnet:packer-1512502276-986baf0 - -ENV GOLANG_VERSION 1.17.2 - -RUN mkdir -p /home/travis/go && chown -R travis /home/travis/go - -ENV GOPATH /home/travis/go - -USER travis - -COPY flake.sh /usr/local/bin/flake.sh - -ENTRYPOINT [ "bash", "flake.sh" ] diff --git a/build-support/docker/flake.sh b/build-support/docker/flake.sh deleted file mode 100644 index 0a885015d8..0000000000 --- a/build-support/docker/flake.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -LOG_FILE="test.log" - -cd $GOPATH/$APP - -export PATH=$GOPATH/$APP/bin:$GOPATH/bin:$PATH - -if ! [[ $(ls -l | grep 'GNUmakefile\|README.md\|LICENSE') ]] ; then - echo "App source not present in cwd. Exiting..." - exit 1 -fi - -mv $TEST_BINARY $TEST_PKG/$TEST_BINARY -cd $TEST_PKG - -for ((i=0; i < $ITERATIONS; i++)) ; do - echo "$(date +"%F %T") - ($((i+1))/$ITERATIONS)" - - ./$TEST_BINARY -test.run "$TEST" -test.parallel 4 -test.timeout 8m -test.v &> $LOG_FILE - echo $? > exit-code - - grep -A10 'panic: ' $LOG_FILE || true - awk '/^[^[:space:]]/ {do_print=0} /--- SKIP/ {do_print=1} do_print==1 {print}' $LOG_FILE - awk '/^[^[:space:]]/ {do_print=0} /--- FAIL/ {do_print=1} do_print==1 {print}' $LOG_FILE - - if [ $(cat exit-code) != "0" ] ; then - exit 1; - fi -done \ No newline at end of file diff --git a/build-support/scripts/test-flake.sh b/build-support/scripts/test-flake.sh deleted file mode 100755 index 743cf7c710..0000000000 --- a/build-support/scripts/test-flake.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null -SCRIPT_DIR=$(pwd) -pushd ../.. > /dev/null -SOURCE_DIR=$(pwd) -pushd build-support/docker > /dev/null -IMG_DIR=$(pwd) -popd > /dev/null - -source "${SCRIPT_DIR}/functions.sh" - -IMAGE="travis-img-v0.13" -CONTAINER="travis-cnt" -GOOS="linux" -GOARCH="amd64" -TEST_BINARY="flake.test" - -function usage { -cat <<-EOF -Usage: test-flake [] - -Description: - - test-flake surfaces flakiness in tests by constraining CPU resources. - - Single or package-wide tests are run for multiple iterations with a configurable - amount of CPU resources. - - 0.15 CPUs and 30 iterations are configured as reasonable defaults. - - See Docker docs for more info on tuning 'cpus' param: - https://docs.docker.com/config/containers/resource_constraints/#cpu - -Options: - - --pkg="" Target package - --test="" Target test (requires pkg flag) - --cpus=0.15 Amount of CPU resources for container - --n=30 Number of times to run tests - -Examples: - - ./test-flake.sh --pkg connect/proxy - ./test-flake.sh --pkg connect/proxy --cpus 0.20 - ./test-flake.sh --pkg connect/proxy --test Listener - ./test-flake.sh --pkg connect/proxy --test TestUpstreamListener - ./test-flake.sh --pkg connect/proxy --test TestUpstreamListener -n 100 -EOF -} - -function build_repro_env { - # Arguments: - # $1 - pkg, Target package - # $2 - test, Target tests - # $3 - cpus, Amount of CPU resources for container - # $4 - n, Number of times to run tests - - APP=$(pwd | awk '{n=split($0, a, "/"); print a[n]}') - - status_stage -e "App:\t\t$APP" - status_stage -e "Package:\t$1" - status_stage -e "Test:\t\t$2" - status_stage -e "CPUs:\t\t$3" - status_stage -e "Iterations:\t$4" - echo - - status_stage "----> Cleaning up old containers..." - if docker ps -a | grep $CONTAINER ; then - docker rm $(docker ps -a | grep $CONTAINER | awk '{print $1;}') - fi - - status_stage '----> Rebuilding image...' - (cd $IMG_DIR && docker build -q -t $IMAGE --no-cache -f Test-Flake.dockerfile .) - - status_stage "--> Building app binary..." - env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/$APP - - status_stage "-> Building test binary..." - env GOOS=$GOOS GOARCH=$GOARCH go test -c "./$1" -o $TEST_BINARY - - status_stage "> Running container..." - status_stage - - docker run \ - --rm \ - --name $CONTAINER \ - --cpus="$3" \ - -v $SOURCE_DIR:/home/travis/go/$APP \ - -e TEST_BINARY="$TEST_BINARY" \ - -e TEST_PKG="$1" \ - -e TEST="$2" \ - -e ITERATIONS="$4" \ - -e APP="$APP" \ - $IMAGE -} - -function err_usage { - err "$1" - err "" - err "$(usage)" -} - -function main { - declare pkg="" - declare test="" - declare cpus="" - declare n="" - - - while test $# -gt 0 - do - case "$1" in - -h | --help ) - usage - return 0 - ;; - - --pkg ) - if test -z "$2" - then - err_usage "ERROR: option pkg requires an argument" - return 1 - fi - - pkg="$2" - shift 2 - ;; - - --test ) - test="$2" - shift 2 - ;; - - --cpus ) - if test -n "$2" - then - cpus="$2" - else - cpus="0.15" - fi - - shift 2 - ;; - - --n ) - if test -n "$2" - then - n="$2" - else - n="30" - fi - - shift 2 - ;; - - * ) - err_usage "ERROR: Unknown argument: '$1'" - return 1 - ;; - esac - done - - build_repro_env "${pkg}" "${test}" "${cpus}" "${n}" || return 1 - - return 0 -} - -main "$@" -exit $? \ No newline at end of file