chore: run nightly tests one by one

fixes: #4585
fixes: #4586

_assets/ci/Jenkinsfile.tests

@@ -116,19 +116,19 @@ pipeline {
       script { env.PKG_URL = "${currentBuild.absoluteUrl}/consoleText" }
       script {
         if (isTestNightlyJob()) {
-          archiveArtifacts artifacts: '**/report.xml, **/test.log'
+          archiveArtifacts artifacts: '**/report_*.xml, **/test_*.log'
 
-          def totalSize = sh(script: "find . -name 'report.xml' | xargs wc -c | tail -n1 | awk '{print \$1}'", returnStdout: true).trim()
-          echo "Total size of all report.xml files: ${totalSize} bytes"
+          def totalSize = sh(script: "find . -name 'report_*.xml' | xargs wc -c | tail -n1 | awk '{print \$1}'", returnStdout: true).trim()
+          echo "Total size of all report_*.xml files: ${totalSize} bytes"
         }
         if (params.UNIT_TEST_RERUN_FAILS) {
-          def rerunReports = findFiles(glob: '**/report_rerun_fails.txt')
+          def rerunReports = findFiles(glob: '**/report_rerun_fails_*.txt')
           if (rerunReports.length > 0) {
-            archiveArtifacts artifacts: '**/report_rerun_fails.txt'
+            archiveArtifacts artifacts: '**/report_rerun_fails_*.txt'
           }
         }
       }
-      junit testResults:          '**/report.xml',
+      junit testResults:          '**/report_*.xml',
             skipOldReports:       true,
             skipPublishingChecks: true,
             skipMarkingBuildUnstable: true

_assets/scripts/run_unit_tests.sh

@@ -23,14 +23,13 @@ if [[ -z "${UNIT_TEST_COUNT}" ]]; then
   UNIT_TEST_COUNT=1
 fi
 
-UNIT_TEST_PACKAGE_TIMEOUT="$((UNIT_TEST_COUNT * 2))m"
-UNIT_TEST_PACKAGE_TIMEOUT_EXTENDED="$((UNIT_TEST_COUNT * 30))m"
+UNIT_TEST_PACKAGE_TIMEOUT="2m"
+UNIT_TEST_PACKAGE_TIMEOUT_EXTENDED="30m"
 
 redirect_stdout() {
   output_file=$1
 
-  if [[ "${CI}" == 'true' ]];
-  then
+  if [[ "${CI}" == 'true' ]]; then
     cat > "${output_file}";
   else
     tee "${output_file}";
@@ -47,12 +46,12 @@ has_extended_timeout() {
   return 1
 }
 
-last_failing_exit_code=0
-
-for package in ${UNIT_TEST_PACKAGES}; do
-  echo -e "${GRN}Testing:${RST} ${package} Count:${UNIT_TEST_COUNT}"
+run_test_for_package() {
+  local package=$1
+  local iteration=$2
+  echo -e "${GRN}Testing:${RST} ${package} Iteration:${iteration}"
   package_dir=$(go list -f "{{.Dir}}" "${package}")
-  output_file=${package_dir}/test.log
+  output_file="${package_dir}/test_${iteration}.log"
 
   if has_extended_timeout "${package}"; then
     package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT_EXTENDED}"
@@ -60,30 +59,42 @@ for package in ${UNIT_TEST_PACKAGES}; do
     package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT}"
   fi
 
+  local report_file="${package_dir}/report_${iteration}.xml"
+  local rerun_report_file="${package_dir}/report_rerun_fails_${iteration}.txt"
+
   gotestsum_flags="${GOTESTSUM_EXTRAFLAGS}"
   if [[ "${CI}" == 'true' ]]; then
-    gotestsum_flags="${gotestsum_flags} --junitfile=${package_dir}/report.xml --rerun-fails-report=${package_dir}/report_rerun_fails.txt"
+    gotestsum_flags="${gotestsum_flags} --junitfile=${report_file} --rerun-fails-report=${rerun_report_file}"
   fi
 
   gotestsum --packages="${package}" ${gotestsum_flags} -- \
     -v ${GOTEST_EXTRAFLAGS} \
     -timeout "${package_timeout}" \
-    -count "${UNIT_TEST_COUNT}" \
+    -count 1 \
     -tags "${BUILD_TAGS}" | \
     redirect_stdout "${output_file}"
-  go_test_exit=$?
+  return $?
+}
 
-  if [[ "${go_test_exit}" -ne 0 ]]; then
-    if [[ "${CI}" == 'true' ]]; then
-      echo -e "${YLW}Failed, see the log:${RST} ${BLD}${output_file}${RST}"
+last_failing_exit_code=0
+
+for package in ${UNIT_TEST_PACKAGES}; do
+  for ((i=1; i<=UNIT_TEST_COUNT; i++)); do
+    run_test_for_package "${package}" "${i}"
+    go_test_exit=$?
+
+    if [[ "${go_test_exit}" -ne 0 ]]; then
+      if [[ "${CI}" == 'true' ]]; then
+        echo -e "${YLW}Failed, see the log:${RST} ${BLD}${output_file}${RST}"
+      fi
+
+      if [[ "$UNIT_TEST_FAILFAST" == 'true' ]]; then
+        exit "${go_test_exit}"
+      fi
+
+      last_failing_exit_code="${go_test_exit}"
     fi
-
-    if [[ "$UNIT_TEST_FAILFAST" == 'true' ]]; then
-      exit "${go_test_exit}"
-    fi
-
-    last_failing_exit_code="${go_test_exit}"
-  fi
+  done
 done
 
 if [[ "${last_failing_exit_code}" -ne 0 ]]; then

_assets/scripts/test_stats.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 
 test_stats = defaultdict(lambda: defaultdict(int))
 
-for file in glob.glob("**/report.xml", recursive=True):
+for file in glob.glob("**/report_*.xml", recursive=True):
     tree = ET.parse(file)
     root = tree.getroot()
     for testcase in root.iter("testcase"):