From fdd979096c2ac46b59b9553357e64b6047703935 Mon Sep 17 00:00:00 2001 From: Patryk Osmaczko Date: Mon, 26 Feb 2024 16:04:20 +0100 Subject: [PATCH] chore: add capability to rerun failed tests - Use `gotestsum` to generate junit test reports and re-run failed tests. - Archive re-run reports in `${package_dir}/report_rerun_fails.txt`. --- Makefile | 2 ++ _assets/ci/Jenkinsfile.tests | 36 +++++++++++++++++++++++++------ _assets/scripts/run_unit_tests.sh | 26 ++++++++++++++++------ shell.nix | 2 +- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index b1105e22f..4c5a4b549 100644 --- a/Makefile +++ b/Makefile @@ -315,6 +315,8 @@ test: test-unit ##@tests Run basic, short tests during development test-unit: export BUILD_TAGS ?= test-unit: export UNIT_TEST_FAILFAST ?= true +test-unit: export UNIT_TEST_RERUN_FAILS ?= true +test-unit: export UNIT_TEST_USE_DEVELOPMENT_LOGGER ?= true # Ensure 'waku' and 'wakuv2' tests are executed first to reduce the impact of flaky tests. # Otherwise, the entire target might fail at the end, making re-runs time-consuming. test-unit: export UNIT_TEST_PACKAGES ?= $(shell go list ./... | \ diff --git a/_assets/ci/Jenkinsfile.tests b/_assets/ci/Jenkinsfile.tests index 1b6acf88b..91e1987ff 100644 --- a/_assets/ci/Jenkinsfile.tests +++ b/_assets/ci/Jenkinsfile.tests @@ -20,6 +20,16 @@ pipeline { defaultValue: getDefaultUnitTestFailfast(), description: 'Should the job fail fast on first test failure?' ) + booleanParam( + name: 'UNIT_TEST_RERUN_FAILS', + defaultValue: getDefaultUnitTestRerunFails(), + description: 'Should the job rerun failed tests?' + ) + booleanParam( + name: 'UNIT_TEST_USE_DEVELOPMENT_LOGGER', + defaultValue: getDefaultUnitTestUseDevelopmentLogger(), + description: 'Should the job use detailed logging for tests, potentially generating large logs?' + ) } options { @@ -103,9 +113,24 @@ pipeline { post { always { script { env.PKG_URL = "${currentBuild.absoluteUrl}/consoleText" } + script { + if (isTestNightlyJob()) { + archiveArtifacts artifacts: '**/report.xml, **/test.log' + + def totalSize = sh(script: "find . -name 'report.xml' | xargs wc -c | tail -n1 | awk '{print \$1}'", returnStdout: true).trim() + echo "Total size of all report.xml files: ${totalSize} bytes" + } + if (params.UNIT_TEST_RERUN_FAILS) { + def rerunReports = findFiles(glob: '**/report_rerun_fails.txt') + if (rerunReports.length > 0) { + archiveArtifacts artifacts: '**/report_rerun_fails.txt' + } + } + } junit testResults: '**/report.xml', skipOldReports: true, - skipPublishingChecks: true + skipPublishingChecks: true, + skipMarkingBuildUnstable: true publishHTML target: [ allowMissing: true, alwaysLinkToLastBuild: true, @@ -115,11 +140,6 @@ pipeline { reportName: 'Reports', reportTitles: 'Test Stats' ] - script { - if (isTestNightlyJob()) { - archiveArtifacts artifacts: '**/report.xml, **/test.log' - } - } } success { script { github.notifyPR(true) } } failure { script { github.notifyPR(false) } } @@ -134,6 +154,10 @@ def isTestNightlyJob() { env.JOB_BASE_NAME == 'tests-nightly' } def getDefaultUnitTestCount() { isTestNightlyJob() ? '20' : '1' } +def getDefaultUnitTestRerunFails() { isTestNightlyJob() ? false : true } + +def getDefaultUnitTestUseDevelopmentLogger() { isTestNightlyJob() ? false : true } + def getDefaultUnitTestFailfast() { isTestNightlyJob() ? false : true } def getDefaultTimeout() { isTestNightlyJob() ? 5*60 : 40 } diff --git a/_assets/scripts/run_unit_tests.sh b/_assets/scripts/run_unit_tests.sh index a33197682..4d9ca174a 100755 --- a/_assets/scripts/run_unit_tests.sh +++ b/_assets/scripts/run_unit_tests.sh @@ -5,10 +5,20 @@ GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) source "${GIT_ROOT}/_assets/scripts/colors.sh" -if [[ $UNIT_TEST_FAILFAST == 'true' ]]; then +if [[ $UNIT_TEST_RERUN_FAILS == 'true' ]]; then + GOTESTSUM_EXTRAFLAGS="${GOTESTSUM_EXTRAFLAGS} --rerun-fails" +elif [[ $UNIT_TEST_FAILFAST == 'true' ]]; then GOTEST_EXTRAFLAGS="${GOTEST_EXTRAFLAGS} -failfast" fi +if [[ $UNIT_TEST_USE_DEVELOPMENT_LOGGER == 'false' ]]; then + if [[ -z $BUILD_TAGS ]]; then + BUILD_TAGS="test_silent" + else + BUILD_TAGS="${BUILD_TAGS},test_silent" + fi +fi + if [[ -z "${UNIT_TEST_COUNT}" ]]; then UNIT_TEST_COUNT=1 fi @@ -40,7 +50,7 @@ has_extended_timeout() { last_failing_exit_code=0 for package in ${UNIT_TEST_PACKAGES}; do - echo -e "${GRN}Testing:${RST} ${package}" + echo -e "${GRN}Testing:${RST} ${package} Count:${UNIT_TEST_COUNT}" package_dir=$(go list -f "{{.Dir}}" "${package}") output_file=${package_dir}/test.log @@ -50,17 +60,19 @@ for package in ${UNIT_TEST_PACKAGES}; do package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT}" fi - go test "${package}" -v ${GOTEST_EXTRAFLAGS} \ + gotestsum_flags="${GOTESTSUM_EXTRAFLAGS}" + if [[ "${CI}" == 'true' ]]; then + gotestsum_flags="${gotestsum_flags} --junitfile=${package_dir}/report.xml --rerun-fails-report=${package_dir}/report_rerun_fails.txt" + fi + + gotestsum --packages="${package}" ${gotestsum_flags} -- \ + -v ${GOTEST_EXTRAFLAGS} \ -timeout "${package_timeout}" \ -count "${UNIT_TEST_COUNT}" \ -tags "${BUILD_TAGS}" | \ redirect_stdout "${output_file}" go_test_exit=$? - if [[ "${CI}" == 'true' ]]; then - go-junit-report -in "${output_file}" -out "${package_dir}"/report.xml - fi - if [[ "${go_test_exit}" -ne 0 ]]; then if [[ "${CI}" == 'true' ]]; then echo -e "${YLW}Failed, see the log:${RST} ${BLD}${output_file}${RST}" diff --git a/shell.nix b/shell.nix index 4c8811848..2ef2a1ac0 100644 --- a/shell.nix +++ b/shell.nix @@ -61,7 +61,7 @@ in pkgs.mkShell { buildInputs = with pkgs; [ git jq which go_1_19 golangci-lint go-junit-report gopls go-bindata gomobileMod - mockgen protobuf3_20 protoc-gen-go + mockgen protobuf3_20 protoc-gen-go gotestsum ] ++ lib.optional stdenv.isDarwin xcodeWrapper; shellHook = lib.optionalString (!isMacM1) ''