From d794e433476db879f34893a61254bd501abdabb2 Mon Sep 17 00:00:00 2001 From: Igor Sirotin Date: Tue, 24 Sep 2024 16:33:26 +0100 Subject: [PATCH] feat_: functional tests coverage (#5805) * feat_: functional tests coverage * fix_: codecov yaml --- .codecov.yml | 41 ++++++++++-- .gitignore | 3 + Makefile | 11 ++-- _assets/ci/Jenkinsfile.tests-rpc | 22 ++++++- _assets/scripts/codecov.sh | 30 +++++++++ _assets/scripts/run_integration_tests.sh | 64 +++++++++++++++++++ _assets/scripts/run_unit_tests.sh | 16 ++--- .../docker-compose.test.status-go.yml | 54 ++++++++++++++-- integration-tests/pytest.ini | 2 +- 9 files changed, 211 insertions(+), 32 deletions(-) create mode 100755 _assets/scripts/codecov.sh create mode 100755 _assets/scripts/run_integration_tests.sh diff --git a/.codecov.yml b/.codecov.yml index f594aba0a..4bd982eb8 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,13 +1,46 @@ +# When modifying this file, please validate using: +# make codecov-validate + +codecov: + require_ci_to_pass: false + notify: + wait_for_ci: true + coverage: - require_ci_to_pass: no - wait_for_ci: no status: project: default: informational: true + unit-tests: + target: auto + flags: + - unit + functional-tests: + target: auto + flags: + - functional patch: default: informational: true + unit-tests: + target: auto + flags: + - unit + functional-tests: + target: auto + flags: + - functional -# When modifying this file, please validate using: -# make codecov-validate +flags: + unit-tests: + paths: + - ".*" + carryforward: false + functional-tests: + paths: + - ".*" + carryforward: true + +comment: + behavior: default + layout: diff,flags,tree diff --git a/.gitignore b/.gitignore index 5e4b8e1d7..419696f64 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,6 @@ __pycache__/ .pytest_cache/ .envrc report/results.xml +integration-tests/coverage +integration-tests/reports +integration-tests/*.log diff --git a/Makefile b/Makefile index 7f2b35afc..10816b3ae 100644 --- a/Makefile +++ b/Makefile @@ -496,18 +496,15 @@ test-verif-proxy-wrapper: CGO_CFLAGS="$(CGO_CFLAGS)" go test -v github.com/status-im/status-go/rpc -tags gowaku_skip_migrations,nimbus_light_client -run ^TestProxySuite$$ -testify.m TestRun -ldflags $(LDFLAGS) -run-integration-tests: SHELL := /bin/sh -run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(shell id -u $$USER) +run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(call sh, id -u) +run-integration-tests: export INTEGRATION_TESTS_REPORT_CODECOV ?= false run-integration-tests: - docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml up -d --build --remove-orphans; \ - docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml logs -f tests-rpc; \ - exit_code=$$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}'); \ - docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml down; \ - exit $$exit_code + @./_assets/scripts/run_integration_tests.sh run-anvil: SHELL := /bin/sh run-anvil: docker-compose -f integration-tests/docker-compose.anvil.yml up --remove-orphans +codecov-validate: SHELL := /bin/sh codecov-validate: curl -X POST --data-binary @.codecov.yml https://codecov.io/validate diff --git a/_assets/ci/Jenkinsfile.tests-rpc b/_assets/ci/Jenkinsfile.tests-rpc index d5d5635d4..7f3987c9e 100644 --- a/_assets/ci/Jenkinsfile.tests-rpc +++ b/_assets/ci/Jenkinsfile.tests-rpc @@ -10,6 +10,11 @@ pipeline { defaultValue: 'develop', description: 'Name of branch to build.' ) + booleanParam( + name: 'INTEGRATION_TESTS_REPORT_CODECOV', + defaultValue: true, + description: 'Should the job report test coverage to Codecov?' + ) } options { @@ -28,12 +33,23 @@ pipeline { environment { PLATFORM = 'tests-rpc' PKG_URL = "${currentBuild.absoluteUrl}/consoleText" + + /* Hack-fix for params not being set in env on first job run. */ + BRANCH = "${params.BRANCH}" + INTEGRATION_TESTS_REPORT_CODECOV = "${params.INTEGRATION_TESTS_REPORT_CODECOV}" } stages { stage('RPC Tests') { steps { script { - sh 'make run-integration-tests' + withCredentials([ + string( + credentialsId: 'codecov-repository-upload-token', + variable: 'CODECOV_TOKEN' + ), + ]) { + nix.shell('make run-integration-tests', pure: false) + } } } } } // stages @@ -42,11 +58,11 @@ pipeline { always { script { archiveArtifacts( - artifacts: '**/results.xml', + artifacts: 'integration-tests/reports/*.xml, integration-tests/*.log, integration-tests/coverage/coverage.html', allowEmptyArchive: true, ) junit( - testResults: '**/results.xml', + testResults: 'integration-tests/reports/*.xml', skipOldReports: true, skipPublishingChecks: true, skipMarkingBuildUnstable: true, diff --git a/_assets/scripts/codecov.sh b/_assets/scripts/codecov.sh new file mode 100755 index 000000000..939725b07 --- /dev/null +++ b/_assets/scripts/codecov.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +source "${GIT_ROOT}/_assets/scripts/colors.sh" + +report_to_codecov() { + # https://go.dev/blog/integration-test-coverage + echo -e "${GRN}Uploading coverage report to Codecov${RST}" + + local tests_report_wildcard="${1}" + local coverage_report="${2}" + local flag="${3}" + + # Gather report files with given wildcard + local report_files_args="" + for file in ${tests_report_wildcard}; do + report_files_args+="--file ${file} " + done + + codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${report_files_args} + codecov upload-process --token "${CODECOV_TOKEN}" -f ${coverage_report} -F "${flag}" +} + +convert_coverage_to_html() { + echo -e "${GRN}Generating HTML coverage report${RST}" + + local input_coverage_report="${1}" + local output_coverage_report="${2}" + + go tool cover -html "${input_coverage_report}" -o "${output_coverage_report}" +} \ No newline at end of file diff --git a/_assets/scripts/run_integration_tests.sh b/_assets/scripts/run_integration_tests.sh new file mode 100755 index 000000000..0a3583ca5 --- /dev/null +++ b/_assets/scripts/run_integration_tests.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -o nounset + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) +source "${GIT_ROOT}/_assets/scripts/colors.sh" +source "${GIT_ROOT}/_assets/scripts/codecov.sh" + +echo -e "${GRN}Running integration tests${RST}" + +root_path="${GIT_ROOT}/integration-tests" +coverage_reports_path="${root_path}/coverage" +binary_coverage_reports_path="${coverage_reports_path}/binary" +merged_coverage_reports_path="${coverage_reports_path}/merged" +test_results_path="${root_path}/reports" + +# Cleanup any previous coverage reports +rm -rf "${coverage_reports_path}" +rm -rf "${test_results_path}" + +# Create directories +mkdir -p "${binary_coverage_reports_path}" +mkdir -p "${merged_coverage_reports_path}" +mkdir -p "${test_results_path}" + +all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml" + +# Run integration tests +echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)" +docker-compose ${all_compose_files} up -d --build --remove-orphans + +echo -e "${GRN}Running tests-rpc${RST}" # Follow the logs, wait for them to finish +docker-compose ${all_compose_files} logs -f tests-rpc > "${root_path}/tests-rpc.log" + +# Stop containers +echo -e "${GRN}Stopping docker containers${RST}" +docker-compose ${all_compose_files} stop + +# Save logs +echo -e "${GRN}Saving logs${RST}" +docker-compose ${all_compose_files} logs status-go > "${root_path}/statusd.log" +docker-compose ${all_compose_files} logs status-go-no-funds > "${root_path}/statusd-no-funds.log" + +# Retrieve exit code +exit_code=$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}'); + +# Cleanup containers +echo -e "${GRN}Removing docker containers${RST}" +docker-compose ${all_compose_files} down + +# Collect coverage reports +echo -e "${GRN}Collecting code coverage reports${RST}" +full_coverage_profile="${coverage_reports_path}/coverage.out" +go tool covdata merge -i="${binary_coverage_reports_path}" -o="${merged_coverage_reports_path}" +go tool covdata textfmt -i="${merged_coverage_reports_path}" -o="${full_coverage_profile}" +convert_coverage_to_html "${full_coverage_profile}" "${coverage_reports_path}/coverage.html" + +# Upload reports to Codecov +if [[ ${INTEGRATION_TESTS_REPORT_CODECOV} == 'true' ]]; then + report_to_codecov "${test_results_path}/*.xml" "${full_coverage_profile}" "functional" +fi + +echo -e "${GRN}Testing finished${RST}" +exit $exit_code \ No newline at end of file diff --git a/_assets/scripts/run_unit_tests.sh b/_assets/scripts/run_unit_tests.sh index c8b6d55b7..ddd971ecf 100755 --- a/_assets/scripts/run_unit_tests.sh +++ b/_assets/scripts/run_unit_tests.sh @@ -2,8 +2,8 @@ set -o pipefail GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) - source "${GIT_ROOT}/_assets/scripts/colors.sh" +source "${GIT_ROOT}/_assets/scripts/codecov.sh" if [[ $UNIT_TEST_RERUN_FAILS == 'true' ]]; then GOTESTSUM_EXTRAFLAGS="${GOTESTSUM_EXTRAFLAGS} --rerun-fails" @@ -82,7 +82,7 @@ run_test_for_packages() { # Merge package coverage results go run ./cmd/test-coverage-utils/gocovmerge.go ${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.* > ${coverage_file} - rm -f "${COVERAGE_REPORTS_DIR}/coverage.out.rerun.*" + rm -f "${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.*" echo "${go_test_exit}" > "${exit_code_file}" if [[ "${go_test_exit}" -ne 0 ]]; then @@ -153,8 +153,7 @@ echo -e "${GRN}Filtering test coverage packages:${RST} ./cmd" grep -v '^github.com/status-im/status-go/cmd/' ${merged_coverage_report} > ${final_coverage_report} # Generate HTML coverage report -echo -e "${GRN}Generating HTML coverage report${RST}" -go tool cover -html ${final_coverage_report} -o test-coverage.html +convert_coverage_to_html ${final_coverage_report} "test-coverage.html" # Upload coverage report to CodeClimate if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then @@ -166,14 +165,7 @@ if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then fi if [[ $UNIT_TEST_REPORT_CODECOV == 'true' ]]; then - echo -e "${GRN}Uploading coverage report to Codecov${RST}" - # https://docs.codeclimate.com/docs/jenkins#jenkins-ci-builds - codecov_report_files_args="" - for file in report_*.xml; do - codecov_report_files_args+="--file ${file} " - done - codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${codecov_report_files_args} - codecov --token "${CODECOV_TOKEN}" -f ${final_coverage_report} -F "unit" + report_to_codecov "report_*.xml" ${final_coverage_report} "unit" fi # Generate report with test stats diff --git a/integration-tests/docker-compose.test.status-go.yml b/integration-tests/docker-compose.test.status-go.yml index 8ec96b268..19215a496 100644 --- a/integration-tests/docker-compose.test.status-go.yml +++ b/integration-tests/docker-compose.test.status-go.yml @@ -1,13 +1,27 @@ services: status-go: + user: ${INTEGRATION_TESTS_DOCKER_UID} build: context: ../ dockerfile: _assets/build/Dockerfile args: build_tags: gowaku_no_rln build_target: statusd - build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true" - entrypoint: ["statusd", "-c", "/static/configs/config.json", "--server=0.0.0.0:8354", "--seed-phrase=test test test test test test test test test test test junk", "--password=Strong12345"] + build_flags: -cover + -ldflags=" + -X github.com/status-im/status-go/params.Version= + -X github.com/status-im/status-go/params.GitCommit=11f83780d + -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ + -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true + " + entrypoint: [ + "statusd", + "-c", "/static/configs/config.json", + "--server", "0.0.0.0:8354", + "--seed-phrase", "test test test test test test test test test test test junk", + "--password", "Strong12345", + "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed. + ] # ports: # - 3333:3333 # - 8354:8354 # use for local debbuging only @@ -16,16 +30,35 @@ services: interval: 5s timeout: 2s retries: 120 + environment: + GOCOVERDIR: "/coverage/binary" + volumes: + - ./coverage/binary:/coverage/binary + stop_signal: SIGINT + # TODO: Remove this duplication when implemented: https://github.com/status-im/status-go/issues/5803 status-go-no-funds: + user: ${INTEGRATION_TESTS_DOCKER_UID} build: context: ../ dockerfile: _assets/build/Dockerfile args: build_tags: gowaku_no_rln build_target: statusd - build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true" - entrypoint: ["statusd", "-c", "/static/configs/config.json", "--seed-phrase=test test test test test test test test test test test takoe", "--password=Strong12345"] + build_flags: -cover + -ldflags=" + -X github.com/status-im/status-go/params.Version= + -X github.com/status-im/status-go/params.GitCommit=11f83780d + -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ + -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true + " + entrypoint: [ + "statusd", + "-c", "/static/configs/config.json", + "--seed-phrase", "test test test test test test test test test test test takoe", + "--password", "Strong12345", + "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed. + ] # ports: # - 3334:3333 # use for local debbuging only healthcheck: @@ -33,6 +66,11 @@ services: interval: 5s timeout: 2s retries: 120 + environment: + GOCOVERDIR: "/coverage/binary" + volumes: + - ./coverage/binary:/coverage/binary + stop_signal: SIGINT tests-rpc: user: ${INTEGRATION_TESTS_DOCKER_UID} @@ -46,6 +84,12 @@ services: build: context: . dockerfile: Dockerfile.tests-rpc - entrypoint: ["pytest", "-m", "wallet", "--rpc_url=http://status-go:3333", "--rpc_url_2=http://status-go-no-funds:3333"] + entrypoint: [ + "pytest", + "-m", "wallet", + "--rpc_url=http://status-go:3333", + "--rpc_url_2=http://status-go-no-funds:3333", + "--junitxml=/tests-rpc/reports/report.xml", + ] volumes: - .:/tests-rpc diff --git a/integration-tests/pytest.ini b/integration-tests/pytest.ini index 198647e22..8af58c3f3 100644 --- a/integration-tests/pytest.ini +++ b/integration-tests/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -s -v --tb=short --junitxml=results.xml +addopts = -s -v --tb=short log_cli=true log_level=INFO