chore: add capability to rerun failed tests

- Use `gotestsum` to generate junit test reports and re-run failed tests.
- Archive re-run reports in `${package_dir}/report_rerun_fails.txt`.

Makefile

@@ -315,6 +315,8 @@ test: test-unit ##@tests Run basic, short tests during development
 
 test-unit: export BUILD_TAGS ?=
 test-unit: export UNIT_TEST_FAILFAST ?= true
+test-unit: export UNIT_TEST_RERUN_FAILS ?= true
+test-unit: export UNIT_TEST_USE_DEVELOPMENT_LOGGER ?= true
 # Ensure 'waku' and 'wakuv2' tests are executed first to reduce the impact of flaky tests.
 # Otherwise, the entire target might fail at the end, making re-runs time-consuming.
 test-unit: export UNIT_TEST_PACKAGES ?= $(shell go list ./... | \

_assets/ci/Jenkinsfile.tests

@@ -20,6 +20,16 @@ pipeline {
       defaultValue: getDefaultUnitTestFailfast(),
       description: 'Should the job fail fast on first test failure?'
     )
+    booleanParam(
+      name: 'UNIT_TEST_RERUN_FAILS',
+      defaultValue: getDefaultUnitTestRerunFails(),
+      description: 'Should the job rerun failed tests?'
+    )
+    booleanParam(
+      name: 'UNIT_TEST_USE_DEVELOPMENT_LOGGER',
+      defaultValue: getDefaultUnitTestUseDevelopmentLogger(),
+      description: 'Should the job use detailed logging for tests, potentially generating large logs?'
+    )
   }
 
   options {
@@ -103,9 +113,24 @@ pipeline {
   post {
     always  {
       script { env.PKG_URL = "${currentBuild.absoluteUrl}/consoleText" }
+      script {
+        if (isTestNightlyJob()) {
+          archiveArtifacts artifacts: '**/report.xml, **/test.log'
+
+          def totalSize = sh(script: "find . -name 'report.xml' | xargs wc -c | tail -n1 | awk '{print \$1}'", returnStdout: true).trim()
+          echo "Total size of all report.xml files: ${totalSize} bytes"
+        }
+        if (params.UNIT_TEST_RERUN_FAILS) {
+          def rerunReports = findFiles(glob: '**/report_rerun_fails.txt')
+          if (rerunReports.length > 0) {
+            archiveArtifacts artifacts: '**/report_rerun_fails.txt'
+          }
+        }
+      }
       junit testResults:          '**/report.xml',
             skipOldReports:       true,
-            skipPublishingChecks: true
+            skipPublishingChecks: true,
+            skipMarkingBuildUnstable: true
       publishHTML target: [
         allowMissing:           true,
         alwaysLinkToLastBuild:  true,
@@ -115,11 +140,6 @@ pipeline {
         reportName:             'Reports',
         reportTitles:           'Test Stats'
       ]
-      script {
-        if (isTestNightlyJob()) {
-          archiveArtifacts artifacts: '**/report.xml, **/test.log'
-        }
-      }
     }
     success { script { github.notifyPR(true) } }
     failure { script { github.notifyPR(false) } }
@@ -134,6 +154,10 @@ def isTestNightlyJob() { env.JOB_BASE_NAME == 'tests-nightly' }
 
 def getDefaultUnitTestCount() { isTestNightlyJob() ? '20' : '1' }
 
+def getDefaultUnitTestRerunFails() { isTestNightlyJob() ? false : true }
+
+def getDefaultUnitTestUseDevelopmentLogger() { isTestNightlyJob() ? false : true }
+
 def getDefaultUnitTestFailfast() { isTestNightlyJob() ? false : true }
 
 def getDefaultTimeout() { isTestNightlyJob() ? 5*60 : 40 }

_assets/scripts/run_unit_tests.sh

@@ -5,10 +5,20 @@ GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
 
 source "${GIT_ROOT}/_assets/scripts/colors.sh"
 
-if [[ $UNIT_TEST_FAILFAST == 'true' ]]; then
+if [[ $UNIT_TEST_RERUN_FAILS == 'true' ]]; then
+  GOTESTSUM_EXTRAFLAGS="${GOTESTSUM_EXTRAFLAGS} --rerun-fails"
+elif [[ $UNIT_TEST_FAILFAST == 'true' ]]; then
   GOTEST_EXTRAFLAGS="${GOTEST_EXTRAFLAGS} -failfast"
 fi
 
+if [[ $UNIT_TEST_USE_DEVELOPMENT_LOGGER == 'false' ]]; then
+  if [[ -z $BUILD_TAGS ]]; then
+    BUILD_TAGS="test_silent"
+  else
+    BUILD_TAGS="${BUILD_TAGS},test_silent"
+  fi
+fi
+
 if [[ -z "${UNIT_TEST_COUNT}" ]]; then
   UNIT_TEST_COUNT=1
 fi
@@ -40,7 +50,7 @@ has_extended_timeout() {
 last_failing_exit_code=0
 
 for package in ${UNIT_TEST_PACKAGES}; do
-  echo -e "${GRN}Testing:${RST} ${package}"
+  echo -e "${GRN}Testing:${RST} ${package} Count:${UNIT_TEST_COUNT}"
   package_dir=$(go list -f "{{.Dir}}" "${package}")
   output_file=${package_dir}/test.log
 
@@ -50,17 +60,19 @@ for package in ${UNIT_TEST_PACKAGES}; do
     package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT}"
   fi
 
-  go test "${package}" -v ${GOTEST_EXTRAFLAGS} \
+  gotestsum_flags="${GOTESTSUM_EXTRAFLAGS}"
+  if [[ "${CI}" == 'true' ]]; then
+    gotestsum_flags="${gotestsum_flags} --junitfile=${package_dir}/report.xml --rerun-fails-report=${package_dir}/report_rerun_fails.txt"
+  fi
+
+  gotestsum --packages="${package}" ${gotestsum_flags} -- \
+    -v ${GOTEST_EXTRAFLAGS} \
     -timeout "${package_timeout}" \
     -count "${UNIT_TEST_COUNT}" \
     -tags "${BUILD_TAGS}" | \
     redirect_stdout "${output_file}"
   go_test_exit=$?
 
-  if [[ "${CI}" == 'true' ]]; then
-    go-junit-report -in "${output_file}" -out "${package_dir}"/report.xml
-  fi
-
   if [[ "${go_test_exit}" -ne 0 ]]; then
     if [[ "${CI}" == 'true' ]]; then
       echo -e "${YLW}Failed, see the log:${RST} ${BLD}${output_file}${RST}"

shell.nix

@@ -61,7 +61,7 @@ in pkgs.mkShell {
   buildInputs = with pkgs; [
     git jq which
     go_1_19 golangci-lint go-junit-report gopls go-bindata gomobileMod
-    mockgen protobuf3_20 protoc-gen-go
+    mockgen protobuf3_20 protoc-gen-go gotestsum
   ] ++ lib.optional stdenv.isDarwin xcodeWrapper;
 
   shellHook = lib.optionalString (!isMacM1) ''