mirror of https://github.com/status-im/consul.git
build-support: remove test-flake machinery
This machinery was not used, and does not appear to be maintained. In practice we really don't need anything to detect flaky tests. Our CI system identifies flaky tests at https://app.circleci.com/insights/github/hashicorp/consul/workflows/go-tests/tests?branch=main Mostly what we need is a way to reproduce flakes, which can be done directly with the Go CLI, using the -race, -count, and (new in Go 1.17) -shuffle flags.
This commit is contained in:
Daniel Nephin
parent
846ff9a2c7
commit
d46f2429ec
|
|
@ -231,9 +231,6 @@ test-internal:
|
|||
test-race:
|
||||
$(MAKE) GOTEST_FLAGS=-race
|
||||
|
||||
test-flake: other-consul lint
|
||||
@$(SHELL) $(CURDIR)/build-support/scripts/test-flake.sh --pkg "$(FLAKE_PKG)" --test "$(FLAKE_TEST)" --cpus "$(FLAKE_CPUS)" --n "$(FLAKE_N)"
|
||||
|
||||
test-docker: linux go-build-image
|
||||
@# -ti run in the foreground showing stdout
|
||||
@# --rm removes the container once its finished running
|
||||
|
|
@ -372,6 +369,6 @@ envoy-regen:
|
|||
@find "command/connect/envoy/testdata" -name '*.golden' -delete
|
||||
@go test -tags '$(GOTAGS)' ./command/connect/envoy -update
|
||||
|
||||
.PHONY: all bin dev dist cov test test-flake test-internal cover lint ui static-assets tools
|
||||
.PHONY: all bin dev dist cov test test-internal cover lint ui static-assets tools
|
||||
.PHONY: docker-images go-build-image ui-build-image static-assets-docker consul-docker ui-docker
|
||||
.PHONY: version proto test-envoy-integ
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
FROM travisci/ci-garnet:packer-1512502276-986baf0
|
||||
|
||||
ENV GOLANG_VERSION 1.17.2
|
||||
|
||||
RUN mkdir -p /home/travis/go && chown -R travis /home/travis/go
|
||||
|
||||
ENV GOPATH /home/travis/go
|
||||
|
||||
USER travis
|
||||
|
||||
COPY flake.sh /usr/local/bin/flake.sh
|
||||
|
||||
ENTRYPOINT [ "bash", "flake.sh" ]
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/bin/bash
|
||||
LOG_FILE="test.log"
|
||||
|
||||
cd $GOPATH/$APP
|
||||
|
||||
export PATH=$GOPATH/$APP/bin:$GOPATH/bin:$PATH
|
||||
|
||||
if ! [[ $(ls -l | grep 'GNUmakefile\|README.md\|LICENSE') ]] ; then
|
||||
echo "App source not present in cwd. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mv $TEST_BINARY $TEST_PKG/$TEST_BINARY
|
||||
cd $TEST_PKG
|
||||
|
||||
for ((i=0; i < $ITERATIONS; i++)) ; do
|
||||
echo "$(date +"%F %T") - ($((i+1))/$ITERATIONS)"
|
||||
|
||||
./$TEST_BINARY -test.run "$TEST" -test.parallel 4 -test.timeout 8m -test.v &> $LOG_FILE
|
||||
echo $? > exit-code
|
||||
|
||||
grep -A10 'panic: ' $LOG_FILE || true
|
||||
awk '/^[^[:space:]]/ {do_print=0} /--- SKIP/ {do_print=1} do_print==1 {print}' $LOG_FILE
|
||||
awk '/^[^[:space:]]/ {do_print=0} /--- FAIL/ {do_print=1} do_print==1 {print}' $LOG_FILE
|
||||
|
||||
if [ $(cat exit-code) != "0" ] ; then
|
||||
exit 1;
|
||||
fi
|
||||
done
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
#!/bin/bash
|
||||
pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null
|
||||
SCRIPT_DIR=$(pwd)
|
||||
pushd ../.. > /dev/null
|
||||
SOURCE_DIR=$(pwd)
|
||||
pushd build-support/docker > /dev/null
|
||||
IMG_DIR=$(pwd)
|
||||
popd > /dev/null
|
||||
|
||||
source "${SCRIPT_DIR}/functions.sh"
|
||||
|
||||
IMAGE="travis-img-v0.13"
|
||||
CONTAINER="travis-cnt"
|
||||
GOOS="linux"
|
||||
GOARCH="amd64"
|
||||
TEST_BINARY="flake.test"
|
||||
|
||||
function usage {
|
||||
cat <<-EOF
|
||||
Usage: test-flake [<options ...>]
|
||||
|
||||
Description:
|
||||
|
||||
test-flake surfaces flakiness in tests by constraining CPU resources.
|
||||
|
||||
Single or package-wide tests are run for multiple iterations with a configurable
|
||||
amount of CPU resources.
|
||||
|
||||
0.15 CPUs and 30 iterations are configured as reasonable defaults.
|
||||
|
||||
See Docker docs for more info on tuning 'cpus' param:
|
||||
https://docs.docker.com/config/containers/resource_constraints/#cpu
|
||||
|
||||
Options:
|
||||
|
||||
--pkg="" Target package
|
||||
--test="" Target test (requires pkg flag)
|
||||
--cpus=0.15 Amount of CPU resources for container
|
||||
--n=30 Number of times to run tests
|
||||
|
||||
Examples:
|
||||
|
||||
./test-flake.sh --pkg connect/proxy
|
||||
./test-flake.sh --pkg connect/proxy --cpus 0.20
|
||||
./test-flake.sh --pkg connect/proxy --test Listener
|
||||
./test-flake.sh --pkg connect/proxy --test TestUpstreamListener
|
||||
./test-flake.sh --pkg connect/proxy --test TestUpstreamListener -n 100
|
||||
EOF
|
||||
}
|
||||
|
||||
function build_repro_env {
|
||||
# Arguments:
|
||||
# $1 - pkg, Target package
|
||||
# $2 - test, Target tests
|
||||
# $3 - cpus, Amount of CPU resources for container
|
||||
# $4 - n, Number of times to run tests
|
||||
|
||||
APP=$(pwd | awk '{n=split($0, a, "/"); print a[n]}')
|
||||
|
||||
status_stage -e "App:\t\t$APP"
|
||||
status_stage -e "Package:\t$1"
|
||||
status_stage -e "Test:\t\t$2"
|
||||
status_stage -e "CPUs:\t\t$3"
|
||||
status_stage -e "Iterations:\t$4"
|
||||
echo
|
||||
|
||||
status_stage "----> Cleaning up old containers..."
|
||||
if docker ps -a | grep $CONTAINER ; then
|
||||
docker rm $(docker ps -a | grep $CONTAINER | awk '{print $1;}')
|
||||
fi
|
||||
|
||||
status_stage '----> Rebuilding image...'
|
||||
(cd $IMG_DIR && docker build -q -t $IMAGE --no-cache -f Test-Flake.dockerfile .)
|
||||
|
||||
status_stage "--> Building app binary..."
|
||||
env GOOS=$GOOS GOARCH=$GOARCH go build -o bin/$APP
|
||||
|
||||
status_stage "-> Building test binary..."
|
||||
env GOOS=$GOOS GOARCH=$GOARCH go test -c "./$1" -o $TEST_BINARY
|
||||
|
||||
status_stage "> Running container..."
|
||||
status_stage
|
||||
|
||||
docker run \
|
||||
--rm \
|
||||
--name $CONTAINER \
|
||||
--cpus="$3" \
|
||||
-v $SOURCE_DIR:/home/travis/go/$APP \
|
||||
-e TEST_BINARY="$TEST_BINARY" \
|
||||
-e TEST_PKG="$1" \
|
||||
-e TEST="$2" \
|
||||
-e ITERATIONS="$4" \
|
||||
-e APP="$APP" \
|
||||
$IMAGE
|
||||
}
|
||||
|
||||
function err_usage {
|
||||
err "$1"
|
||||
err ""
|
||||
err "$(usage)"
|
||||
}
|
||||
|
||||
function main {
|
||||
declare pkg=""
|
||||
declare test=""
|
||||
declare cpus=""
|
||||
declare n=""
|
||||
|
||||
|
||||
while test $# -gt 0
|
||||
do
|
||||
case "$1" in
|
||||
-h | --help )
|
||||
usage
|
||||
return 0
|
||||
;;
|
||||
|
||||
--pkg )
|
||||
if test -z "$2"
|
||||
then
|
||||
err_usage "ERROR: option pkg requires an argument"
|
||||
return 1
|
||||
fi
|
||||
|
||||
pkg="$2"
|
||||
shift 2
|
||||
;;
|
||||
|
||||
--test )
|
||||
test="$2"
|
||||
shift 2
|
||||
;;
|
||||
|
||||
--cpus )
|
||||
if test -n "$2"
|
||||
then
|
||||
cpus="$2"
|
||||
else
|
||||
cpus="0.15"
|
||||
fi
|
||||
|
||||
shift 2
|
||||
;;
|
||||
|
||||
--n )
|
||||
if test -n "$2"
|
||||
then
|
||||
n="$2"
|
||||
else
|
||||
n="30"
|
||||
fi
|
||||
|
||||
shift 2
|
||||
;;
|
||||
|
||||
* )
|
||||
err_usage "ERROR: Unknown argument: '$1'"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
build_repro_env "${pkg}" "${test}" "${cpus}" "${n}" || return 1
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit $?
|
||||
Loading…
Reference in New Issue