feat_: functional tests coverage (#5805)

* feat_: functional tests coverage

* fix_: codecov yaml

.codecov.yml

@@ -1,13 +1,46 @@
+# When modifying this file, please validate using:
+#  make codecov-validate
+
+codecov:
+  require_ci_to_pass: false
+  notify:
+    wait_for_ci: true
+
 coverage:
-  require_ci_to_pass: no
-  wait_for_ci: no
   status:
     project:
       default:
         informational: true
+      unit-tests:
+        target: auto
+        flags:
+          - unit
+      functional-tests:
+        target: auto
+        flags:
+          - functional
     patch:
       default:
         informational: true
+      unit-tests:
+        target: auto
+        flags:
+          - unit
+      functional-tests:
+        target: auto
+        flags:
+          - functional
 
-# When modifying this file, please validate using:
+flags:
-# make codecov-validate
+  unit-tests:
+    paths:
+      - ".*"
+    carryforward: false
+  functional-tests:
+    paths:
+      - ".*"
+    carryforward: true
+
+comment:
+  behavior: default
+  layout: diff,flags,tree

.gitignore

@@ -108,3 +108,6 @@ __pycache__/
 .pytest_cache/
 .envrc
 report/results.xml
+integration-tests/coverage
+integration-tests/reports
+integration-tests/*.log

Makefile

@@ -496,18 +496,15 @@ test-verif-proxy-wrapper:
 	CGO_CFLAGS="$(CGO_CFLAGS)" go test -v github.com/status-im/status-go/rpc -tags gowaku_skip_migrations,nimbus_light_client -run ^TestProxySuite$$ -testify.m TestRun -ldflags $(LDFLAGS)
 
 
-run-integration-tests: SHELL := /bin/sh
+run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(call sh, id -u)
-run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(shell id -u $$USER)
+run-integration-tests: export INTEGRATION_TESTS_REPORT_CODECOV ?= false
 run-integration-tests:
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml up -d --build --remove-orphans; \
+	@./_assets/scripts/run_integration_tests.sh
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml logs -f tests-rpc; \
-	exit_code=$$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}'); \
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml down; \
-	exit $$exit_code
 
 run-anvil: SHELL := /bin/sh
 run-anvil:
 	docker-compose -f integration-tests/docker-compose.anvil.yml up --remove-orphans
 
+codecov-validate: SHELL := /bin/sh
 codecov-validate:
 	curl -X POST --data-binary @.codecov.yml https://codecov.io/validate

_assets/ci/Jenkinsfile.tests-rpc

@@ -10,6 +10,11 @@ pipeline {
       defaultValue: 'develop',
       description: 'Name of branch to build.'
     )
+    booleanParam(
+      name: 'INTEGRATION_TESTS_REPORT_CODECOV',
+      defaultValue: true,
+      description: 'Should the job report test coverage to Codecov?'
+    )
   }
 
   options {
@@ -28,12 +33,23 @@ pipeline {
   environment {
     PLATFORM = 'tests-rpc'
     PKG_URL = "${currentBuild.absoluteUrl}/consoleText"
+
+    /* Hack-fix for params not being set in env on first job run. */
+    BRANCH =                           "${params.BRANCH}"
+    INTEGRATION_TESTS_REPORT_CODECOV = "${params.INTEGRATION_TESTS_REPORT_CODECOV}"
   }
 
   stages {
     stage('RPC Tests') {
       steps { script {
-        sh 'make run-integration-tests'
+        withCredentials([
+          string(
+            credentialsId: 'codecov-repository-upload-token',
+            variable: 'CODECOV_TOKEN'
+          ),
+        ]) {
+          nix.shell('make run-integration-tests', pure: false)
+        }
       } }
     }
   } // stages
@@ -42,11 +58,11 @@ pipeline {
     always {
       script {
         archiveArtifacts(
-          artifacts: '**/results.xml',
+          artifacts: 'integration-tests/reports/*.xml, integration-tests/*.log, integration-tests/coverage/coverage.html',
           allowEmptyArchive: true,
         )
         junit(
-          testResults: '**/results.xml',
+          testResults: 'integration-tests/reports/*.xml',
           skipOldReports: true,
           skipPublishingChecks: true,
           skipMarkingBuildUnstable: true,

_assets/scripts/codecov.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+
+report_to_codecov() {
+  # https://go.dev/blog/integration-test-coverage
+  echo -e "${GRN}Uploading coverage report to Codecov${RST}"
+
+  local tests_report_wildcard="${1}"
+  local coverage_report="${2}"
+  local flag="${3}"
+
+  # Gather report files with given wildcard
+  local report_files_args=""
+  for file in ${tests_report_wildcard}; do
+    report_files_args+="--file ${file} "
+  done
+
+  codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${report_files_args}
+  codecov upload-process --token "${CODECOV_TOKEN}" -f ${coverage_report} -F "${flag}"
+}
+
+convert_coverage_to_html() {
+  echo -e "${GRN}Generating HTML coverage report${RST}"
+
+  local input_coverage_report="${1}"
+  local output_coverage_report="${2}"
+
+  go tool cover -html "${input_coverage_report}" -o "${output_coverage_report}"
+}

_assets/scripts/run_integration_tests.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -o nounset
+
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
+
+echo -e "${GRN}Running integration tests${RST}"
+
+root_path="${GIT_ROOT}/integration-tests"
+coverage_reports_path="${root_path}/coverage"
+binary_coverage_reports_path="${coverage_reports_path}/binary"
+merged_coverage_reports_path="${coverage_reports_path}/merged"
+test_results_path="${root_path}/reports"
+
+# Cleanup any previous coverage reports
+rm -rf "${coverage_reports_path}"
+rm -rf "${test_results_path}"
+
+# Create directories
+mkdir -p "${binary_coverage_reports_path}"
+mkdir -p "${merged_coverage_reports_path}"
+mkdir -p "${test_results_path}"
+
+all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml"
+
+# Run integration tests
+echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)"
+docker-compose ${all_compose_files} up -d --build --remove-orphans
+
+echo -e "${GRN}Running tests-rpc${RST}" # Follow the logs, wait for them to finish
+docker-compose ${all_compose_files} logs -f tests-rpc > "${root_path}/tests-rpc.log"
+
+# Stop containers
+echo -e "${GRN}Stopping docker containers${RST}"
+docker-compose ${all_compose_files} stop
+
+# Save logs
+echo -e "${GRN}Saving logs${RST}"
+docker-compose ${all_compose_files} logs status-go > "${root_path}/statusd.log"
+docker-compose ${all_compose_files} logs status-go-no-funds > "${root_path}/statusd-no-funds.log"
+
+# Retrieve exit code
+exit_code=$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}');
+
+# Cleanup containers
+echo -e "${GRN}Removing docker containers${RST}"
+docker-compose ${all_compose_files} down
+
+# Collect coverage reports
+echo -e "${GRN}Collecting code coverage reports${RST}"
+full_coverage_profile="${coverage_reports_path}/coverage.out"
+go tool covdata merge -i="${binary_coverage_reports_path}" -o="${merged_coverage_reports_path}"
+go tool covdata textfmt -i="${merged_coverage_reports_path}" -o="${full_coverage_profile}"
+convert_coverage_to_html "${full_coverage_profile}" "${coverage_reports_path}/coverage.html"
+
+# Upload reports to Codecov
+if [[ ${INTEGRATION_TESTS_REPORT_CODECOV} == 'true' ]]; then
+  report_to_codecov "${test_results_path}/*.xml" "${full_coverage_profile}" "functional"
+fi
+
+echo -e "${GRN}Testing finished${RST}"
+exit $exit_code

_assets/scripts/run_unit_tests.sh

@@ -2,8 +2,8 @@
 set -o pipefail
 
 GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
-
 source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
 
 if [[ $UNIT_TEST_RERUN_FAILS == 'true' ]]; then
   GOTESTSUM_EXTRAFLAGS="${GOTESTSUM_EXTRAFLAGS} --rerun-fails"
@@ -82,7 +82,7 @@ run_test_for_packages() {
 
   # Merge package coverage results
   go run ./cmd/test-coverage-utils/gocovmerge.go ${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.* > ${coverage_file}
-  rm -f "${COVERAGE_REPORTS_DIR}/coverage.out.rerun.*"
+  rm -f "${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.*"
 
   echo "${go_test_exit}" > "${exit_code_file}"
   if [[ "${go_test_exit}" -ne 0 ]]; then
@@ -153,8 +153,7 @@ echo -e "${GRN}Filtering test coverage packages:${RST} ./cmd"
 grep -v '^github.com/status-im/status-go/cmd/' ${merged_coverage_report} > ${final_coverage_report}
 
 # Generate HTML coverage report
-echo -e "${GRN}Generating HTML coverage report${RST}"
+convert_coverage_to_html ${final_coverage_report} "test-coverage.html"
-go tool cover -html ${final_coverage_report} -o test-coverage.html
 
 # Upload coverage report to CodeClimate
 if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then
@@ -166,14 +165,7 @@ if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then
 fi
 
 if [[ $UNIT_TEST_REPORT_CODECOV == 'true' ]]; then
-  echo -e "${GRN}Uploading coverage report to Codecov${RST}"
+  report_to_codecov "report_*.xml" ${final_coverage_report} "unit"
-  # https://docs.codeclimate.com/docs/jenkins#jenkins-ci-builds
-  codecov_report_files_args=""
-  for file in report_*.xml; do
-    codecov_report_files_args+="--file ${file} "
-  done
-  codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${codecov_report_files_args}
-  codecov --token "${CODECOV_TOKEN}" -f ${final_coverage_report} -F "unit"
 fi
 
 # Generate report with test stats

integration-tests/docker-compose.test.status-go.yml

@@ -1,13 +1,27 @@
 services:
   status-go:
+    user: ${INTEGRATION_TESTS_DOCKER_UID}
     build:
       context: ../
       dockerfile: _assets/build/Dockerfile
       args:
         build_tags: gowaku_no_rln
         build_target: statusd
-        build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true"
+        build_flags: -cover
-    entrypoint: ["statusd", "-c", "/static/configs/config.json", "--server=0.0.0.0:8354", "--seed-phrase=test test test test test test test test test test test junk", "--password=Strong12345"]
+          -ldflags="
+            -X github.com/status-im/status-go/params.Version=
+            -X github.com/status-im/status-go/params.GitCommit=11f83780d
+            -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/
+            -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true
+          "
+    entrypoint: [
+      "statusd",
+      "-c", "/static/configs/config.json",
+      "--server", "0.0.0.0:8354",
+      "--seed-phrase", "test test test test test test test test test test test junk",
+      "--password", "Strong12345",
+      "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed.
+    ]
     # ports:
     #   - 3333:3333
     #   - 8354:8354 # use for local debbuging only
@@ -16,16 +30,35 @@ services:
       interval: 5s
       timeout: 2s
       retries: 120
+    environment:
+      GOCOVERDIR: "/coverage/binary"
+    volumes:
+      - ./coverage/binary:/coverage/binary
+    stop_signal: SIGINT
 
+  # TODO: Remove this duplication when implemented: https://github.com/status-im/status-go/issues/5803
   status-go-no-funds:
+    user: ${INTEGRATION_TESTS_DOCKER_UID}
     build:
       context: ../
       dockerfile: _assets/build/Dockerfile
       args:
         build_tags: gowaku_no_rln
         build_target: statusd
-        build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true"
+        build_flags: -cover
-    entrypoint: ["statusd", "-c", "/static/configs/config.json", "--seed-phrase=test test test test test test test test test test test takoe", "--password=Strong12345"]
+          -ldflags="
+          -X github.com/status-im/status-go/params.Version=
+          -X github.com/status-im/status-go/params.GitCommit=11f83780d
+          -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/
+          -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true
+          "
+    entrypoint: [
+      "statusd",
+      "-c", "/static/configs/config.json",
+      "--seed-phrase", "test test test test test test test test test test test takoe",
+      "--password", "Strong12345",
+      "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed.
+    ]
     # ports:
     #   - 3334:3333 # use for local debbuging only
     healthcheck:
@@ -33,6 +66,11 @@ services:
       interval: 5s
       timeout: 2s
       retries: 120
+    environment:
+      GOCOVERDIR: "/coverage/binary"
+    volumes:
+      - ./coverage/binary:/coverage/binary
+    stop_signal: SIGINT
 
   tests-rpc:
     user: ${INTEGRATION_TESTS_DOCKER_UID}
@@ -46,6 +84,12 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.tests-rpc
-    entrypoint: ["pytest", "-m", "wallet", "--rpc_url=http://status-go:3333", "--rpc_url_2=http://status-go-no-funds:3333"]
+    entrypoint: [
+      "pytest",
+      "-m", "wallet",
+      "--rpc_url=http://status-go:3333",
+      "--rpc_url_2=http://status-go-no-funds:3333",
+      "--junitxml=/tests-rpc/reports/report.xml",
+    ]
     volumes:
       - .:/tests-rpc

integration-tests/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = -s -v --tb=short --junitxml=results.xml
+addopts = -s -v --tb=short
 
 log_cli=true
 log_level=INFO