feat_: functional tests coverage (#5805)

* feat_: functional tests coverage * fix_: codecov yaml
2024-09-24 16:33:26 +01:00 · 2024-09-24 16:33:26 +01:00 · d794e43347
parent 77ef8f1fb7
commit d794e43347
9 changed files with 211 additions and 32 deletions
--- a/.codecov.yml
+++ b/.codecov.yml
@ -1,13 +1,46 @@
+# When modifying this file, please validate using:
+#  make codecov-validate
+
+codecov:
+  require_ci_to_pass: false
+  notify:
+    wait_for_ci: true
+
 coverage:
-  require_ci_to_pass: no
-  wait_for_ci: no
  status:
    project:
      default:
        informational: true
+      unit-tests:
+        target: auto
+        flags:
+          - unit
+      functional-tests:
+        target: auto
+        flags:
+          - functional
    patch:
      default:
        informational: true
+      unit-tests:
+        target: auto
+        flags:
+          - unit
+      functional-tests:
+        target: auto
+        flags:
+          - functional

-# When modifying this file, please validate using:
-# make codecov-validate
+flags:
+  unit-tests:
+    paths:
+      - ".*"
+    carryforward: false
+  functional-tests:
+    paths:
+      - ".*"
+    carryforward: true
+
+comment:
+  behavior: default
+  layout: diff,flags,tree
--- a/.gitignore
+++ b/.gitignore
@ -108,3 +108,6 @@ __pycache__/
 .pytest_cache/
 .envrc
 report/results.xml
+integration-tests/coverage
+integration-tests/reports
+integration-tests/*.log
--- a/11
+++ b/11
@ -496,18 +496,15 @@ test-verif-proxy-wrapper:
 	CGO_CFLAGS="$(CGO_CFLAGS)" go test -v github.com/status-im/status-go/rpc -tags gowaku_skip_migrations,nimbus_light_client -run ^TestProxySuite$$ -testify.m TestRun -ldflags $(LDFLAGS)


-run-integration-tests: SHELL := /bin/sh
-run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(shell id -u $$USER)
+run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(call sh, id -u)
+run-integration-tests: export INTEGRATION_TESTS_REPORT_CODECOV ?= false
 run-integration-tests:
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml up -d --build --remove-orphans; \
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml logs -f tests-rpc; \
-	exit_code=$$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}'); \
-	docker-compose -f integration-tests/docker-compose.anvil.yml -f integration-tests/docker-compose.test.status-go.yml down; \
-	exit $$exit_code
+	@./_assets/scripts/run_integration_tests.sh

 run-anvil: SHELL := /bin/sh
 run-anvil:
 	docker-compose -f integration-tests/docker-compose.anvil.yml up --remove-orphans

+codecov-validate: SHELL := /bin/sh
 codecov-validate:
 	curl -X POST --data-binary @.codecov.yml https://codecov.io/validate
--- a/_assets/ci/Jenkinsfile.tests-rpc
+++ b/_assets/ci/Jenkinsfile.tests-rpc
@ -10,6 +10,11 @@ pipeline {
      defaultValue: 'develop',
      description: 'Name of branch to build.'
    )
+    booleanParam(
+      name: 'INTEGRATION_TESTS_REPORT_CODECOV',
+      defaultValue: true,
+      description: 'Should the job report test coverage to Codecov?'
+    )
  }

  options {
@ -28,12 +33,23 @@ pipeline {
  environment {
    PLATFORM = 'tests-rpc'
    PKG_URL = "${currentBuild.absoluteUrl}/consoleText"
+
+    /* Hack-fix for params not being set in env on first job run. */
+    BRANCH =                           "${params.BRANCH}"
+    INTEGRATION_TESTS_REPORT_CODECOV = "${params.INTEGRATION_TESTS_REPORT_CODECOV}"
  }

  stages {
    stage('RPC Tests') {
      steps { script {
-        sh 'make run-integration-tests'
+        withCredentials([
+          string(
+            credentialsId: 'codecov-repository-upload-token',
+            variable: 'CODECOV_TOKEN'
+          ),
+        ]) {
+          nix.shell('make run-integration-tests', pure: false)
+        }
      } }
    }
  } // stages
@ -42,11 +58,11 @@ pipeline {
    always {
      script {
        archiveArtifacts(
-          artifacts: '**/results.xml',
+          artifacts: 'integration-tests/reports/*.xml, integration-tests/*.log, integration-tests/coverage/coverage.html',
          allowEmptyArchive: true,
        )
        junit(
-          testResults: '**/results.xml',
+          testResults: 'integration-tests/reports/*.xml',
          skipOldReports: true,
          skipPublishingChecks: true,
          skipMarkingBuildUnstable: true,
--- a/_assets/scripts/codecov.sh
+++ b/_assets/scripts/codecov.sh
@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+
+report_to_codecov() {
+  # https://go.dev/blog/integration-test-coverage
+  echo -e "${GRN}Uploading coverage report to Codecov${RST}"
+
+  local tests_report_wildcard="${1}"
+  local coverage_report="${2}"
+  local flag="${3}"
+
+  # Gather report files with given wildcard
+  local report_files_args=""
+  for file in ${tests_report_wildcard}; do
+    report_files_args+="--file ${file} "
+  done
+
+  codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${report_files_args}
+  codecov upload-process --token "${CODECOV_TOKEN}" -f ${coverage_report} -F "${flag}"
+}
+
+convert_coverage_to_html() {
+  echo -e "${GRN}Generating HTML coverage report${RST}"
+
+  local input_coverage_report="${1}"
+  local output_coverage_report="${2}"
+
+  go tool cover -html "${input_coverage_report}" -o "${output_coverage_report}"
+}
--- a/_assets/scripts/run_integration_tests.sh
+++ b/_assets/scripts/run_integration_tests.sh
@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -o nounset
+
+GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
+source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"
+
+echo -e "${GRN}Running integration tests${RST}"
+
+root_path="${GIT_ROOT}/integration-tests"
+coverage_reports_path="${root_path}/coverage"
+binary_coverage_reports_path="${coverage_reports_path}/binary"
+merged_coverage_reports_path="${coverage_reports_path}/merged"
+test_results_path="${root_path}/reports"
+
+# Cleanup any previous coverage reports
+rm -rf "${coverage_reports_path}"
+rm -rf "${test_results_path}"
+
+# Create directories
+mkdir -p "${binary_coverage_reports_path}"
+mkdir -p "${merged_coverage_reports_path}"
+mkdir -p "${test_results_path}"
+
+all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml"
+
+# Run integration tests
+echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)"
+docker-compose ${all_compose_files} up -d --build --remove-orphans
+
+echo -e "${GRN}Running tests-rpc${RST}" # Follow the logs, wait for them to finish
+docker-compose ${all_compose_files} logs -f tests-rpc > "${root_path}/tests-rpc.log"
+
+# Stop containers
+echo -e "${GRN}Stopping docker containers${RST}"
+docker-compose ${all_compose_files} stop
+
+# Save logs
+echo -e "${GRN}Saving logs${RST}"
+docker-compose ${all_compose_files} logs status-go > "${root_path}/statusd.log"
+docker-compose ${all_compose_files} logs status-go-no-funds > "${root_path}/statusd-no-funds.log"
+
+# Retrieve exit code
+exit_code=$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}');
+
+# Cleanup containers
+echo -e "${GRN}Removing docker containers${RST}"
+docker-compose ${all_compose_files} down
+
+# Collect coverage reports
+echo -e "${GRN}Collecting code coverage reports${RST}"
+full_coverage_profile="${coverage_reports_path}/coverage.out"
+go tool covdata merge -i="${binary_coverage_reports_path}" -o="${merged_coverage_reports_path}"
+go tool covdata textfmt -i="${merged_coverage_reports_path}" -o="${full_coverage_profile}"
+convert_coverage_to_html "${full_coverage_profile}" "${coverage_reports_path}/coverage.html"
+
+# Upload reports to Codecov
+if [[ ${INTEGRATION_TESTS_REPORT_CODECOV} == 'true' ]]; then
+  report_to_codecov "${test_results_path}/*.xml" "${full_coverage_profile}" "functional"
+fi
+
+echo -e "${GRN}Testing finished${RST}"
+exit $exit_code
--- a/_assets/scripts/run_unit_tests.sh
+++ b/_assets/scripts/run_unit_tests.sh
@ -2,8 +2,8 @@
 set -o pipefail

 GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
-
 source "${GIT_ROOT}/_assets/scripts/colors.sh"
+source "${GIT_ROOT}/_assets/scripts/codecov.sh"

 if [[ $UNIT_TEST_RERUN_FAILS == 'true' ]]; then
  GOTESTSUM_EXTRAFLAGS="${GOTESTSUM_EXTRAFLAGS} --rerun-fails"
@ -82,7 +82,7 @@ run_test_for_packages() {

  # Merge package coverage results
  go run ./cmd/test-coverage-utils/gocovmerge.go ${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.* > ${coverage_file}
-  rm -f "${COVERAGE_REPORTS_DIR}/coverage.out.rerun.*"
+  rm -f "${TEST_WITH_COVERAGE_REPORTS_DIR}/coverage.out.rerun.*"

  echo "${go_test_exit}" > "${exit_code_file}"
  if [[ "${go_test_exit}" -ne 0 ]]; then
@ -153,8 +153,7 @@ echo -e "${GRN}Filtering test coverage packages:${RST} ./cmd"
 grep -v '^github.com/status-im/status-go/cmd/' ${merged_coverage_report} > ${final_coverage_report}

 # Generate HTML coverage report
-echo -e "${GRN}Generating HTML coverage report${RST}"
-go tool cover -html ${final_coverage_report} -o test-coverage.html
+convert_coverage_to_html ${final_coverage_report} "test-coverage.html"

 # Upload coverage report to CodeClimate
 if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then
@ -166,14 +165,7 @@ if [[ $UNIT_TEST_REPORT_CODECLIMATE == 'true' ]]; then
 fi

 if [[ $UNIT_TEST_REPORT_CODECOV == 'true' ]]; then
-  echo -e "${GRN}Uploading coverage report to Codecov${RST}"
-  # https://docs.codeclimate.com/docs/jenkins#jenkins-ci-builds
-  codecov_report_files_args=""
-  for file in report_*.xml; do
-    codecov_report_files_args+="--file ${file} "
-  done
-  codecov do-upload --token "${CODECOV_TOKEN}" --report-type test_results ${codecov_report_files_args}
-  codecov --token "${CODECOV_TOKEN}" -f ${final_coverage_report} -F "unit"
+  report_to_codecov "report_*.xml" ${final_coverage_report} "unit"
 fi

 # Generate report with test stats
--- a/integration-tests/docker-compose.test.status-go.yml
+++ b/integration-tests/docker-compose.test.status-go.yml
@ -1,13 +1,27 @@
 services:
  status-go:
+    user: ${INTEGRATION_TESTS_DOCKER_UID}
    build:
      context: ../
      dockerfile: _assets/build/Dockerfile
      args:
        build_tags: gowaku_no_rln
        build_target: statusd
-        build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true"
-    entrypoint: ["statusd", "-c", "/static/configs/config.json", "--server=0.0.0.0:8354", "--seed-phrase=test test test test test test test test test test test junk", "--password=Strong12345"]
+        build_flags: -cover
+          -ldflags="
+            -X github.com/status-im/status-go/params.Version=
+            -X github.com/status-im/status-go/params.GitCommit=11f83780d
+            -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/
+            -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true
+          "
+    entrypoint: [
+      "statusd",
+      "-c", "/static/configs/config.json",
+      "--server", "0.0.0.0:8354",
+      "--seed-phrase", "test test test test test test test test test test test junk",
+      "--password", "Strong12345",
+      "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed.
+    ]
    # ports:
    #   - 3333:3333
    #   - 8354:8354 # use for local debbuging only
@ -16,16 +30,35 @@ services:
      interval: 5s
      timeout: 2s
      retries: 120
+    environment:
+      GOCOVERDIR: "/coverage/binary"
+    volumes:
+      - ./coverage/binary:/coverage/binary
+    stop_signal: SIGINT

+  # TODO: Remove this duplication when implemented: https://github.com/status-im/status-go/issues/5803
  status-go-no-funds:
+    user: ${INTEGRATION_TESTS_DOCKER_UID}
    build:
      context: ../
      dockerfile: _assets/build/Dockerfile
      args:
        build_tags: gowaku_no_rln
        build_target: statusd
-        build_flags: -ldflags="-X github.com/status-im/status-go/params.Version= -X github.com/status-im/status-go/params.GitCommit=11f83780d -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/ -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true"
-    entrypoint: ["statusd", "-c", "/static/configs/config.json", "--seed-phrase=test test test test test test test test test test test takoe", "--password=Strong12345"]
+        build_flags: -cover
+          -ldflags="
+          -X github.com/status-im/status-go/params.Version=
+          -X github.com/status-im/status-go/params.GitCommit=11f83780d
+          -X github.com/status-im/status-go/params.IpfsGatewayURL=https://ipfs.status.im/
+          -X github.com/status-im/status-go/vendor/github.com/ethereum/go-ethereum/metrics.EnabledStr=true
+          "
+    entrypoint: [
+      "statusd",
+      "-c", "/static/configs/config.json",
+      "--seed-phrase", "test test test test test test test test test test test takoe",
+      "--password", "Strong12345",
+      "--dir", "/tmp/status-go-data", # Keep in sync with `config.json/DataDir` value. Later this arg will not be needed.
+    ]
    # ports:
    #   - 3334:3333 # use for local debbuging only
    healthcheck:
@ -33,6 +66,11 @@ services:
      interval: 5s
      timeout: 2s
      retries: 120
+    environment:
+      GOCOVERDIR: "/coverage/binary"
+    volumes:
+      - ./coverage/binary:/coverage/binary
+    stop_signal: SIGINT

  tests-rpc:
    user: ${INTEGRATION_TESTS_DOCKER_UID}
@ -46,6 +84,12 @@ services:
    build:
      context: .
      dockerfile: Dockerfile.tests-rpc
-    entrypoint: ["pytest", "-m", "wallet", "--rpc_url=http://status-go:3333", "--rpc_url_2=http://status-go-no-funds:3333"]
+    entrypoint: [
+      "pytest",
+      "-m", "wallet",
+      "--rpc_url=http://status-go:3333",
+      "--rpc_url_2=http://status-go-no-funds:3333",
+      "--junitxml=/tests-rpc/reports/report.xml",
+    ]
    volumes:
      - .:/tests-rpc
--- a/integration-tests/pytest.ini
+++ b/integration-tests/pytest.ini
@ -1,5 +1,5 @@
 [pytest]
-addopts = -s -v --tb=short --junitxml=results.xml
+addopts = -s -v --tb=short

 log_cli=true
 log_level=INFO