diff --git a/.gitignore b/.gitignore index 419696f64..c23ca6b55 100644 --- a/.gitignore +++ b/.gitignore @@ -101,13 +101,13 @@ test-bob/ # Nix /.nix-gcroots/ -# integration-tests +# functional tests __pycache__/ *.py[cod] *$py.class .pytest_cache/ .envrc report/results.xml -integration-tests/coverage -integration-tests/reports -integration-tests/*.log +tests-functional/coverage +tests-functional/reports +tests-functional/*.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1e96f754..3e11fb958 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ Once a feature has undergone testing and is ready for production use, the featur Test coverage is vital for ensuring the stability and reliability of our project. Follow these guidelines: 1. Before submitting a PR, check the existing test coverage for the modified or new code. -2. Ensure that the new code is covered by relevant unit tests, integration tests, or other appropriate testing methods. +2. Ensure that the new code is covered by relevant unit, integration and/or functional tests, or other appropriate testing methods. 3. If modifying existing code, update or add tests to cover the changes. 4. If adding new features, include tests that demonstrate the correct functionality and handle edge cases. diff --git a/Makefile b/Makefile index ba2b037f9..d9baad2f7 100644 --- a/Makefile +++ b/Makefile @@ -399,6 +399,11 @@ test-e2e: ##@tests Run e2e tests test-e2e-race: export GOTEST_EXTRAFLAGS=-race test-e2e-race: test-e2e ##@tests Run e2e tests with -race flag +test-functional: export FUNCTIONAL_TESTS_DOCKER_UID ?= $(call sh, id -u) +test-functional: export FUNCTIONAL_TESTS_REPORT_CODECOV ?= false +test-functional: + @./_assets/scripts/run_functional_tests.sh + canary-test: node-canary # TODO: uncomment that! #_assets/scripts/canary_test_mailservers.sh ./config/cli/fleet-eth.prod.json @@ -505,15 +510,9 @@ build-verif-proxy-wrapper: test-verif-proxy-wrapper: CGO_CFLAGS="$(CGO_CFLAGS)" go test -v github.com/status-im/status-go/rpc -tags gowaku_skip_migrations,nimbus_light_client -run ^TestProxySuite$$ -testify.m TestRun -ldflags $(LDFLAGS) - -run-integration-tests: export INTEGRATION_TESTS_DOCKER_UID ?= $(call sh, id -u) -run-integration-tests: export INTEGRATION_TESTS_REPORT_CODECOV ?= false -run-integration-tests: - @./_assets/scripts/run_integration_tests.sh - run-anvil: SHELL := /bin/sh run-anvil: - docker-compose -f integration-tests/docker-compose.anvil.yml up --remove-orphans + docker-compose -f tests-functional/docker-compose.anvil.yml up --remove-orphans codecov-validate: SHELL := /bin/sh codecov-validate: diff --git a/_assets/build/Dockerfile b/_assets/build/Dockerfile index f7f3a582f..86d341ecb 100644 --- a/_assets/build/Dockerfile +++ b/_assets/build/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p /static/configs COPY --from=builder /go/src/github.com/status-im/status-go/build/bin/$build_target /usr/local/bin/ COPY --from=builder /go/src/github.com/status-im/status-go/static/keys/* /static/keys/ -COPY --from=builder /go/src/github.com/status-im/status-go/integration-tests/config.json /static/configs/ +COPY --from=builder /go/src/github.com/status-im/status-go/tests-functional/config.json /static/configs/ RUN ln -s /usr/local/bin/$build_target /usr/local/bin/entrypoint diff --git a/_assets/ci/Jenkinsfile.tests-rpc b/_assets/ci/Jenkinsfile.tests-rpc index 7f3987c9e..90d01e0eb 100644 --- a/_assets/ci/Jenkinsfile.tests-rpc +++ b/_assets/ci/Jenkinsfile.tests-rpc @@ -11,7 +11,7 @@ pipeline { description: 'Name of branch to build.' ) booleanParam( - name: 'INTEGRATION_TESTS_REPORT_CODECOV', + name: 'FUNCTIONAL_TESTS_REPORT_CODECOV', defaultValue: true, description: 'Should the job report test coverage to Codecov?' ) @@ -36,7 +36,7 @@ pipeline { /* Hack-fix for params not being set in env on first job run. */ BRANCH = "${params.BRANCH}" - INTEGRATION_TESTS_REPORT_CODECOV = "${params.INTEGRATION_TESTS_REPORT_CODECOV}" + FUNCTIONAL_TESTS_REPORT_CODECOV = "${params.FUNCTIONAL_TESTS_REPORT_CODECOV}" } stages { @@ -48,7 +48,7 @@ pipeline { variable: 'CODECOV_TOKEN' ), ]) { - nix.shell('make run-integration-tests', pure: false) + nix.shell('make test-functional', pure: false) } } } } @@ -58,11 +58,11 @@ pipeline { always { script { archiveArtifacts( - artifacts: 'integration-tests/reports/*.xml, integration-tests/*.log, integration-tests/coverage/coverage.html', + artifacts: 'tests-functional/reports/*.xml, tests-functional/*.log, tests-functional/coverage/coverage.html', allowEmptyArchive: true, ) junit( - testResults: 'integration-tests/reports/*.xml', + testResults: 'tests-functional/reports/*.xml', skipOldReports: true, skipPublishingChecks: true, skipMarkingBuildUnstable: true, diff --git a/_assets/scripts/run_integration_tests.sh b/_assets/scripts/run_functional_tests.sh similarity index 88% rename from _assets/scripts/run_integration_tests.sh rename to _assets/scripts/run_functional_tests.sh index 0a3583ca5..24f2c9310 100755 --- a/_assets/scripts/run_integration_tests.sh +++ b/_assets/scripts/run_functional_tests.sh @@ -6,9 +6,9 @@ GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) source "${GIT_ROOT}/_assets/scripts/colors.sh" source "${GIT_ROOT}/_assets/scripts/codecov.sh" -echo -e "${GRN}Running integration tests${RST}" +echo -e "${GRN}Running functional tests${RST}" -root_path="${GIT_ROOT}/integration-tests" +root_path="${GIT_ROOT}/tests-functional" coverage_reports_path="${root_path}/coverage" binary_coverage_reports_path="${coverage_reports_path}/binary" merged_coverage_reports_path="${coverage_reports_path}/merged" @@ -25,7 +25,7 @@ mkdir -p "${test_results_path}" all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml" -# Run integration tests +# Run functional tests echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)" docker-compose ${all_compose_files} up -d --build --remove-orphans @@ -42,7 +42,7 @@ docker-compose ${all_compose_files} logs status-go > "${root_path}/statusd.log" docker-compose ${all_compose_files} logs status-go-no-funds > "${root_path}/statusd-no-funds.log" # Retrieve exit code -exit_code=$(docker inspect integration-tests_tests-rpc_1 -f '{{.State.ExitCode}}'); +exit_code=$(docker inspect tests-functional_tests-rpc_1 -f '{{.State.ExitCode}}'); # Cleanup containers echo -e "${GRN}Removing docker containers${RST}" @@ -56,7 +56,7 @@ go tool covdata textfmt -i="${merged_coverage_reports_path}" -o="${full_coverage convert_coverage_to_html "${full_coverage_profile}" "${coverage_reports_path}/coverage.html" # Upload reports to Codecov -if [[ ${INTEGRATION_TESTS_REPORT_CODECOV} == 'true' ]]; then +if [[ ${FUNCTIONAL_TESTS_REPORT_CODECOV} == 'true' ]]; then report_to_codecov "${test_results_path}/*.xml" "${full_coverage_profile}" "functional" fi diff --git a/_docs/policies/tests.md b/_docs/policies/tests.md index fa5b25c31..9e376ec75 100644 --- a/_docs/policies/tests.md +++ b/_docs/policies/tests.md @@ -9,10 +9,10 @@ - Prove that the functionality performs as described - Can be falsified - Are resistant to fuzzing -- All new `integration tests` MUST BE validated via a minimum of 1000 tests. +- All new `functional tests` MUST BE validated via a minimum of 1000 tests. - This can be achieved using the `-count` or `-test.count` flag with the test command eg: `-count 1000` / `-test.count 1000` - Where the CI can not support this work flow automatically, the developer MUST perform validation tests via local testing. - - `TODO` Add link to issue for CI automation of validation test runs of new `integration tests`. + - `TODO` Add link to issue for CI automation of validation test runs of new `functional tests`. - Ensuring that the test passes consistently every time gives confidence that the test is not flaky. ## Flaky Tests diff --git a/integration-tests/Dockerfile b/tests-functional/Dockerfile similarity index 100% rename from integration-tests/Dockerfile rename to tests-functional/Dockerfile diff --git a/integration-tests/Dockerfile.tests-rpc b/tests-functional/Dockerfile.tests-rpc similarity index 100% rename from integration-tests/Dockerfile.tests-rpc rename to tests-functional/Dockerfile.tests-rpc diff --git a/integration-tests/README.MD b/tests-functional/README.MD similarity index 66% rename from integration-tests/README.MD rename to tests-functional/README.MD index 8799d6050..16ceefa9c 100644 --- a/integration-tests/README.MD +++ b/tests-functional/README.MD @@ -1,6 +1,6 @@ ## Overview -Integration tests for status-go +Functional tests for status-go ## Table of Contents @@ -14,28 +14,28 @@ Integration tests for status-go * Install [Docker](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) * Install [Python 3.10.14](https://www.python.org/downloads/) -* In `integration-tests` directory, run `pip install -r requirements.txt` +* In `./tests-functional`, run `pip install -r requirements.txt` * **Optional (for test development)**: Use Python virtual environment for better dependency management. You can follow the guide [here](https://akrabat.com/creating-virtual-environments-with-pyenv/): ## How to Run ### Running dev RPC (anvil with contracts) -- In `integration-tests` run `docker compose -f docker-compose.anvil.yml up --remove-orphans --build`, as result: +- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml up --remove-orphans --build`, as result: * an [anvil](https://book.getfoundry.sh/reference/anvil/) container with ChainID 31337 exposed on `0.0.0.0:8545` will start running * Status-im contracts will be deployed to the network ### Run tests -- In `integration-tests` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans`, as result: +- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans`, as result: * a container with [status-go as daemon](https://github.com/status-im/status-go/issues/5175) will be created with APIModules exposed on `0.0.0.0:3333` * status-go will use [anvil](https://book.getfoundry.sh/reference/anvil/) as RPCURL with ChainID 31337 * all Status-im contracts will be deployed to the network -* In `integration-tests/tests` directory run `pytest -m wallet` +* In `./tests-functional/tests` directory run `pytest -m wallet` ## Implementation details -- Integration tests are implemented in `integration-tests/tests` based on [pytest](https://docs.pytest.org/en/8.2.x/) +- Functional tests are implemented in `./tests-functional/tests` based on [pytest](https://docs.pytest.org/en/8.2.x/) - Every test has two types of verifications: - `verify_is_valid_json_rpc_response()` checks for status code 200, non-empty response, JSON-RPC structure, presence of the `result` field, and expected ID. - `jsonschema.validate()` is used to check that the response contains expected data, including required fields and types. Schemas are stored in `/schemas/wallet_MethodName` - - New schemas can be generated using `integration-tests/schema_builder.py` by passing a response to the `CustomSchemaBuilder(schema_name).create_schema(response.json())` method, should be used only on test creation phase, please search `how to create schema:` to see an example in a test \ No newline at end of file + - New schemas can be generated using `./tests-functional/schema_builder.py` by passing a response to the `CustomSchemaBuilder(schema_name).create_schema(response.json())` method, should be used only on test creation phase, please search `how to create schema:` to see an example in a test \ No newline at end of file diff --git a/integration-tests/config.json b/tests-functional/config.json similarity index 100% rename from integration-tests/config.json rename to tests-functional/config.json diff --git a/integration-tests/conftest.py b/tests-functional/conftest.py similarity index 100% rename from integration-tests/conftest.py rename to tests-functional/conftest.py diff --git a/integration-tests/docker-compose.anvil.yml b/tests-functional/docker-compose.anvil.yml similarity index 100% rename from integration-tests/docker-compose.anvil.yml rename to tests-functional/docker-compose.anvil.yml diff --git a/integration-tests/docker-compose.status-go.local.yml b/tests-functional/docker-compose.status-go.local.yml similarity index 100% rename from integration-tests/docker-compose.status-go.local.yml rename to tests-functional/docker-compose.status-go.local.yml diff --git a/integration-tests/docker-compose.test.status-go.yml b/tests-functional/docker-compose.test.status-go.yml similarity index 95% rename from integration-tests/docker-compose.test.status-go.yml rename to tests-functional/docker-compose.test.status-go.yml index ae2b01bfd..8df073f79 100644 --- a/integration-tests/docker-compose.test.status-go.yml +++ b/tests-functional/docker-compose.test.status-go.yml @@ -1,6 +1,6 @@ services: status-go: - user: ${INTEGRATION_TESTS_DOCKER_UID} + user: ${FUNCTIONAL_TESTS_DOCKER_UID} build: context: ../ dockerfile: _assets/build/Dockerfile @@ -32,7 +32,7 @@ services: # TODO: Remove this duplication when implemented: https://github.com/status-im/status-go/issues/5803 status-go-no-funds: - user: ${INTEGRATION_TESTS_DOCKER_UID} + user: ${FUNCTIONAL_TESTS_DOCKER_UID} build: context: ../ dockerfile: _assets/build/Dockerfile @@ -59,7 +59,7 @@ services: stop_signal: SIGINT tests-rpc: - user: ${INTEGRATION_TESTS_DOCKER_UID} + user: ${FUNCTIONAL_TESTS_DOCKER_UID} depends_on: status-go: condition: service_healthy diff --git a/integration-tests/entrypoint.sh b/tests-functional/entrypoint.sh similarity index 100% rename from integration-tests/entrypoint.sh rename to tests-functional/entrypoint.sh diff --git a/integration-tests/pytest.ini b/tests-functional/pytest.ini similarity index 100% rename from integration-tests/pytest.ini rename to tests-functional/pytest.ini diff --git a/integration-tests/requirements.txt b/tests-functional/requirements.txt similarity index 100% rename from integration-tests/requirements.txt rename to tests-functional/requirements.txt diff --git a/integration-tests/schema_builder.py b/tests-functional/schema_builder.py similarity index 100% rename from integration-tests/schema_builder.py rename to tests-functional/schema_builder.py diff --git a/integration-tests/schemas/accounts_getKeypairs b/tests-functional/schemas/accounts_getKeypairs similarity index 100% rename from integration-tests/schemas/accounts_getKeypairs rename to tests-functional/schemas/accounts_getKeypairs diff --git a/integration-tests/schemas/accounts_hasPairedDevices b/tests-functional/schemas/accounts_hasPairedDevices similarity index 100% rename from integration-tests/schemas/accounts_hasPairedDevices rename to tests-functional/schemas/accounts_hasPairedDevices diff --git a/integration-tests/schemas/accounts_remainingAccountCapacity b/tests-functional/schemas/accounts_remainingAccountCapacity similarity index 100% rename from integration-tests/schemas/accounts_remainingAccountCapacity rename to tests-functional/schemas/accounts_remainingAccountCapacity diff --git a/integration-tests/schemas/wakuext_peers b/tests-functional/schemas/wakuext_peers similarity index 100% rename from integration-tests/schemas/wakuext_peers rename to tests-functional/schemas/wakuext_peers diff --git a/integration-tests/schemas/wallet_checkRecentHistoryForChainIDs b/tests-functional/schemas/wallet_checkRecentHistoryForChainIDs similarity index 100% rename from integration-tests/schemas/wallet_checkRecentHistoryForChainIDs rename to tests-functional/schemas/wallet_checkRecentHistoryForChainIDs diff --git a/integration-tests/schemas/wallet_createMultiTransaction/transferTx_error b/tests-functional/schemas/wallet_createMultiTransaction/transferTx_error similarity index 100% rename from integration-tests/schemas/wallet_createMultiTransaction/transferTx_error rename to tests-functional/schemas/wallet_createMultiTransaction/transferTx_error diff --git a/integration-tests/schemas/wallet_createMultiTransaction/transferTx_positive b/tests-functional/schemas/wallet_createMultiTransaction/transferTx_positive similarity index 100% rename from integration-tests/schemas/wallet_createMultiTransaction/transferTx_positive rename to tests-functional/schemas/wallet_createMultiTransaction/transferTx_positive diff --git a/integration-tests/schemas/wallet_fetchAllCurrencyFormats b/tests-functional/schemas/wallet_fetchAllCurrencyFormats similarity index 100% rename from integration-tests/schemas/wallet_fetchAllCurrencyFormats rename to tests-functional/schemas/wallet_fetchAllCurrencyFormats diff --git a/integration-tests/schemas/wallet_getCachedCurrencyFormats b/tests-functional/schemas/wallet_getCachedCurrencyFormats similarity index 100% rename from integration-tests/schemas/wallet_getCachedCurrencyFormats rename to tests-functional/schemas/wallet_getCachedCurrencyFormats diff --git a/integration-tests/schemas/wallet_getCryptoOnRamps b/tests-functional/schemas/wallet_getCryptoOnRamps similarity index 100% rename from integration-tests/schemas/wallet_getCryptoOnRamps rename to tests-functional/schemas/wallet_getCryptoOnRamps diff --git a/integration-tests/schemas/wallet_getEthereumChains b/tests-functional/schemas/wallet_getEthereumChains similarity index 100% rename from integration-tests/schemas/wallet_getEthereumChains rename to tests-functional/schemas/wallet_getEthereumChains diff --git a/integration-tests/schemas/wallet_getPendingTransactions b/tests-functional/schemas/wallet_getPendingTransactions similarity index 100% rename from integration-tests/schemas/wallet_getPendingTransactions rename to tests-functional/schemas/wallet_getPendingTransactions diff --git a/integration-tests/schemas/wallet_getPendingTransactionsForIdentities b/tests-functional/schemas/wallet_getPendingTransactionsForIdentities similarity index 100% rename from integration-tests/schemas/wallet_getPendingTransactionsForIdentities rename to tests-functional/schemas/wallet_getPendingTransactionsForIdentities diff --git a/integration-tests/schemas/wallet_getTokenList b/tests-functional/schemas/wallet_getTokenList similarity index 100% rename from integration-tests/schemas/wallet_getTokenList rename to tests-functional/schemas/wallet_getTokenList diff --git a/integration-tests/schemas/wallet_startWallet b/tests-functional/schemas/wallet_startWallet similarity index 100% rename from integration-tests/schemas/wallet_startWallet rename to tests-functional/schemas/wallet_startWallet diff --git a/integration-tests/tests/test_accounts.py b/tests-functional/tests/test_accounts.py similarity index 100% rename from integration-tests/tests/test_accounts.py rename to tests-functional/tests/test_accounts.py diff --git a/integration-tests/tests/test_cases.py b/tests-functional/tests/test_cases.py similarity index 100% rename from integration-tests/tests/test_cases.py rename to tests-functional/tests/test_cases.py diff --git a/integration-tests/tests/test_router.py b/tests-functional/tests/test_router.py similarity index 100% rename from integration-tests/tests/test_router.py rename to tests-functional/tests/test_router.py diff --git a/integration-tests/tests/test_waku_rpc.py b/tests-functional/tests/test_waku_rpc.py similarity index 100% rename from integration-tests/tests/test_waku_rpc.py rename to tests-functional/tests/test_waku_rpc.py diff --git a/integration-tests/tests/test_wallet_rpc.py b/tests-functional/tests/test_wallet_rpc.py similarity index 100% rename from integration-tests/tests/test_wallet_rpc.py rename to tests-functional/tests/test_wallet_rpc.py