chore_: run tests concurrently (#5731)
* chore_: fake flaky tests * chore_: run tests in parallel (wip) * chore_: test-with-coverage-old * chore_: lint-fix fake flaky tests * chore_: cleanup * chore_: * chore_: lint-fix fake flaky tests * chore_: enable running some tests sequentially * chore_: enable back all tests * chore_: no sequential packages * chore_: cleanup * chore_: remove fake flaky packages * chore_: cleanup * chore_: attempt UNIT_TEST_COUNT as test count * chore_: spawn multiple processes * fix_: don't spawn multiple processes * chore_: revert redirect_stdout changes * fix_: update Jenkinsfile * fix_: use env variables * fix_: typo
This commit is contained in:
Igor Sirotin
GitHub
parent
c0d1f11ac2
commit
cc722359b5
8
Makefile
8
Makefile
|
|
@ -374,14 +374,6 @@ test-unit: export UNIT_TEST_PACKAGES ?= $(call sh, go list ./... | grep -E '/wak
|
|||
grep -v /transactions/fake | \
|
||||
grep -E -v '/waku(/.*|$$)' | \
|
||||
grep -E -v '/wakuv2(/.*|$$)')
|
||||
test-unit: export UNIT_TEST_PACKAGES_NOT_PARALLELIZABLE ?= \
|
||||
github.com/status-im/status-go/api \
|
||||
github.com/status-im/status-go/mailserver \
|
||||
github.com/status-im/status-go/multiaccounts/settings \
|
||||
github.com/status-im/status-go/node \
|
||||
github.com/status-im/status-go/services/wakuext
|
||||
test-unit: export UNIT_TEST_PACKAGES_WITH_EXTENDED_TIMEOUT ?= \
|
||||
github.com/status-im/status-go/protocol
|
||||
test-unit: ##@tests Run unit and integration tests
|
||||
./_assets/scripts/run_unit_tests.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -181,18 +181,16 @@ pipeline {
|
|||
env.PKG_URL = "${currentBuild.absoluteUrl}/consoleText"
|
||||
|
||||
if (isTestNightlyJob()) {
|
||||
archiveArtifacts('**/report_*.xml, **/test_*.log')
|
||||
def totalSize = sh(script: "find . -name 'report_*.xml' | xargs wc -c | tail -n1 | awk '{print \$1}'", returnStdout: true).trim()
|
||||
println("Total size of all report_*.xml files: ${totalSize} bytes.")
|
||||
archiveArtifacts('report.xml, test.log')
|
||||
}
|
||||
if (params.UNIT_TEST_RERUN_FAILS) {
|
||||
def rerunReports = findFiles(glob: '**/report_rerun_fails_*.txt')
|
||||
def rerunReports = findFiles(glob: 'report_rerun_fails.txt')
|
||||
if (rerunReports.length > 0) {
|
||||
archiveArtifacts('**/report_rerun_fails_*.txt')
|
||||
archiveArtifacts('report_rerun_fails.txt')
|
||||
}
|
||||
}
|
||||
junit(
|
||||
testResults: '**/report_*.xml',
|
||||
testResults: 'report.xml',
|
||||
skipOldReports: true,
|
||||
skipPublishingChecks: true,
|
||||
skipMarkingBuildUnstable: true
|
||||
|
|
@ -212,7 +210,7 @@ pipeline {
|
|||
failure {
|
||||
script {
|
||||
github.notifyPR(false)
|
||||
archiveArtifacts('**/test_*.log')
|
||||
archiveArtifacts('test.log')
|
||||
}
|
||||
}
|
||||
cleanup {
|
||||
|
|
|
|||
|
|
@ -23,9 +23,6 @@ if [[ -z "${UNIT_TEST_COUNT}" ]]; then
|
|||
UNIT_TEST_COUNT=1
|
||||
fi
|
||||
|
||||
UNIT_TEST_PACKAGE_TIMEOUT="3m"
|
||||
UNIT_TEST_PACKAGE_TIMEOUT_EXTENDED="45m"
|
||||
|
||||
redirect_stdout() {
|
||||
output_file=$1
|
||||
|
||||
|
|
@ -36,43 +33,14 @@ redirect_stdout() {
|
|||
fi
|
||||
}
|
||||
|
||||
has_extended_timeout() {
|
||||
local package
|
||||
for package in ${UNIT_TEST_PACKAGES_WITH_EXTENDED_TIMEOUT}; do
|
||||
if [[ "$1" == "${package}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
run_test_for_packages() {
|
||||
local output_file="test.log"
|
||||
local coverage_file="test.coverage.out"
|
||||
local report_file="report.xml"
|
||||
local rerun_report_file="report_rerun_fails.txt"
|
||||
local exit_code_file="exit_code.txt"
|
||||
|
||||
is_parallelizable() {
|
||||
local package
|
||||
for package in ${UNIT_TEST_PACKAGES_NOT_PARALLELIZABLE}; do
|
||||
if [[ "$1" == "${package}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
run_test_for_package() {
|
||||
local package=$1
|
||||
local iteration=$2
|
||||
echo -e "${GRN}Testing:${RST} ${package} Iteration:${iteration}"
|
||||
package_dir=$(go list -f "{{.Dir}}" "${package}")
|
||||
output_file="${package_dir}/test_${iteration}.log"
|
||||
coverage_file="${package_dir}/test_${iteration}.coverage.out"
|
||||
|
||||
if has_extended_timeout "${package}"; then
|
||||
package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT_EXTENDED}"
|
||||
else
|
||||
package_timeout="${UNIT_TEST_PACKAGE_TIMEOUT}"
|
||||
fi
|
||||
|
||||
local report_file="${package_dir}/report_${iteration}.xml"
|
||||
local rerun_report_file="${package_dir}/report_rerun_fails_${iteration}.txt"
|
||||
local exit_code_file="${package_dir}/exit_code_${iteration}.txt"
|
||||
echo -e "${GRN}Testing:${RST} All packages. Single iteration. -test.count=${UNIT_TEST_COUNT}"
|
||||
|
||||
gotestsum_flags="${GOTESTSUM_EXTRAFLAGS}"
|
||||
if [[ "${CI}" == 'true' ]]; then
|
||||
|
|
@ -80,24 +48,23 @@ run_test_for_package() {
|
|||
fi
|
||||
|
||||
# Cleanup previous coverage reports
|
||||
rm -f ${package_dir}/coverage.out.rerun.*
|
||||
rm -f coverage.out.rerun.*
|
||||
|
||||
PACKAGE_DIR=${package_dir} gotestsum --packages="${package}" ${gotestsum_flags} --raw-command -- \
|
||||
# Run tests
|
||||
gotestsum --packages="${UNIT_TEST_PACKAGES}" ${gotestsum_flags} --raw-command -- \
|
||||
./_assets/scripts/test-with-coverage.sh \
|
||||
${package} \
|
||||
-v ${GOTEST_EXTRAFLAGS} \
|
||||
-timeout "${package_timeout}" \
|
||||
-count 1 \
|
||||
-timeout 45m \
|
||||
-tags "${BUILD_TAGS}" | \
|
||||
redirect_stdout "${output_file}"
|
||||
|
||||
local go_test_exit=$?
|
||||
|
||||
# Merge package coverage results
|
||||
go run ./cmd/test-coverage-utils/gocovmerge.go ${package_dir}/coverage.out.rerun.* > ${coverage_file}
|
||||
go run ./cmd/test-coverage-utils/gocovmerge.go coverage.out.rerun.* > ${coverage_file}
|
||||
|
||||
# Cleanup coverage reports
|
||||
rm -f ${package_dir}/coverage.out.rerun.*
|
||||
rm -f coverage.out.rerun.*
|
||||
|
||||
echo "${go_test_exit}" > "${exit_code_file}"
|
||||
if [[ "${go_test_exit}" -ne 0 ]]; then
|
||||
|
|
@ -113,26 +80,10 @@ if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then
|
|||
cc-test-reporter before-build
|
||||
fi
|
||||
|
||||
echo -e "${GRN}Testing HEAD:${RST} $(git rev-parse HEAD)"
|
||||
|
||||
rm -rf ./**/*.coverage.out
|
||||
|
||||
for package in ${UNIT_TEST_PACKAGES}; do
|
||||
for ((i=1; i<=UNIT_TEST_COUNT; i++)); do
|
||||
if ! is_parallelizable "${package}" || [[ "$UNIT_TEST_FAILFAST" == 'true' ]]; then
|
||||
run_test_for_package "${package}" "${i}"
|
||||
go_test_exit=$?
|
||||
if [[ "$UNIT_TEST_FAILFAST" == 'true' ]]; then
|
||||
if [[ "${go_test_exit}" -ne 0 ]]; then
|
||||
exit "${go_test_exit}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
run_test_for_package "${package}" "${i}" &
|
||||
fi
|
||||
done
|
||||
wait # Wait for all background jobs to finish
|
||||
done
|
||||
echo -e "${GRN}Testing HEAD:${RST} $(git rev-parse HEAD)"
|
||||
run_test_for_packages
|
||||
|
||||
# Gather test coverage results
|
||||
rm -f c.out c-full.out
|
||||
|
|
@ -153,7 +104,7 @@ fi
|
|||
|
||||
shopt -s globstar nullglob # Enable recursive globbing
|
||||
if [[ "${UNIT_TEST_COUNT}" -gt 1 ]]; then
|
||||
for exit_code_file in "${GIT_ROOT}"/**/exit_code_*.txt; do
|
||||
for exit_code_file in "${GIT_ROOT}"/**/exit_code.txt; do
|
||||
read exit_code < "${exit_code_file}"
|
||||
if [[ "${exit_code}" -ne 0 ]]; then
|
||||
mkdir -p "${GIT_ROOT}/reports"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,21 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
coverage_file_path="$(mktemp coverage.out.rerun.XXXXXXXXXX --tmpdir="${PACKAGE_DIR}")"
|
||||
|
||||
packages=""
|
||||
coverage_file_path="$(mktemp coverage.out.rerun.XXXXXXXXXX)"
|
||||
count=1
|
||||
|
||||
# This is a hack to workaround gotestsum behaviour. When using a --raw-command,
|
||||
# gotestsum will only pass the package when rerunning a test. Otherwise we should pass the package ourselves.
|
||||
# https://github.com/gotestyourself/gotestsum/blob/03568ab6d48faabdb632013632ac42687b5f17d1/cmd/main.go#L331-L336
|
||||
if [[ "$*" != *"-test.run"* ]]; then
|
||||
packages="${UNIT_TEST_PACKAGES}"
|
||||
count=${UNIT_TEST_COUNT}
|
||||
fi
|
||||
|
||||
go test -json \
|
||||
${packages} \
|
||||
-count=${count} \
|
||||
-covermode=atomic \
|
||||
-coverprofile="${coverage_file_path}" \
|
||||
-coverpkg ./... \
|
||||
|
|
|
|||
Loading…
Reference in New Issue