#!/usr/bin/env bash set -o nounset GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) source "${GIT_ROOT}/scripts/colors.sh" source "${GIT_ROOT}/scripts/codecov.sh" : "${FUNCTIONAL_TESTS_LOG_LEVEL:=INFO}" : "${FUNCTIONAL_TESTS_REPORT_CODECOV:=false}" : "${FUNCTIONAL_TESTS_BUILD_TAGS:=gowaku_no_rln}" : "${USE_LOGOS_STORAGE:=false}" : "${FUNCTIONAL_TESTS_MARKER:=rpc}" : "${FUNCTIONAL_TESTS_RERUNS:=2}" : "${FUNCTIONAL_TESTS_PARALLEL:=12}" : "${PEER_IMAGES:=}" : "${PEER_REFS:=}" : "${USE_TORRENT:=false}" if [[ "${USE_LOGOS_STORAGE}" == "true" ]]; then export WAKU_STORE_MESSAGE_RETENTION_SECONDS="${WAKU_STORE_MESSAGE_RETENTION_SECONDS:-60}" fi echo -e "${GRN}Running functional tests${RST}" root_path="${GIT_ROOT}/test/functional" coverage_reports_path="${root_path}/coverage" binary_coverage_reports_path="${coverage_reports_path}/binary" merged_coverage_reports_path="${coverage_reports_path}/merged" test_results_path="${root_path}/reports" logs_path="${root_path}/logs" # Cleanup any previous coverage reports rm -rf "${coverage_reports_path}" rm -rf "${test_results_path}" rm -rf "${logs_path}" # Create directories mkdir -p "${binary_coverage_reports_path}" mkdir -p "${merged_coverage_reports_path}" mkdir -p "${test_results_path}" mkdir -p "${logs_path}" all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.waku.yml" identifier=${FUNCTIONAL_TESTS_CONTAINER_PREFIX:-"status-go-func-tests-$(git rev-parse --short HEAD)"} project_name="${identifier,,}" image_name="statusgo-${identifier,,}" dump_compose_logs_done=0 dump_compose_logs() { [[ ${dump_compose_logs_done} -eq 1 ]] && return 0 dump_compose_logs_done=1 { echo "=== docker compose ps ===" docker compose -p "${project_name}" ${all_compose_files} ps -a echo echo "=== docker compose logs ===" docker compose -p "${project_name}" ${all_compose_files} logs --no-color --timestamps } > "${logs_path}/compose.log" 2>&1 || true } trap dump_compose_logs EXIT # Remove orphans echo -e "${GRN}Cleanup old containers${RST}" docker ps -a --filter "name=${project_name}" --filter "status=exited" -q | xargs -r docker rm -f # Build statusgo image echo -e "${GRN}Building status-go${RST}" build_tags="${FUNCTIONAL_TESTS_BUILD_TAGS}" pytest_marker_expr="${FUNCTIONAL_TESTS_MARKER}" if [[ "${USE_LOGOS_STORAGE}" == "true" ]]; then build_tags="${build_tags} use_logos_storage" if [[ -n "${LOGOS_STORAGE_LIB_DIR:-}" && ! -f "${LOGOS_STORAGE_LIB_DIR}/libstorage.so" ]]; then echo -e "${YEL}No libstorage.so at ${LOGOS_STORAGE_LIB_DIR}; build it first with make build-storage.${RST}" fi else pytest_marker_expr="(${pytest_marker_expr}) and not logos_storage" fi if [[ "${USE_TORRENT}" == "true" ]]; then build_tags="${build_tags} use_torrent" fi docker build . \ --build-arg "build_flags=-cover" \ --build-arg "build_tags=${build_tags}" \ --build-arg "use_logos_storage=${USE_LOGOS_STORAGE}" \ --build-arg "use_torrent=${USE_TORRENT}" \ --build-arg "enable_go_cache=false" \ --tag "${image_name}" if [[ $? -ne 0 ]]; then echo -e "${RED}Docker build failed. Exiting.${RST}" exit 1 fi # Run docker echo -e "${GRN}Running status-go external dependencies${RST}" docker compose -p ${project_name} ${all_compose_files} up -d --build --remove-orphans # Wait for wakufleet-scanner to finish before running tests. If it fails, # there's no point in starting tests because wakufleetconfig.json will not # be available for status-backend. echo -e "${GRN}Waiting for wakufleet-scanner to complete${RST}" scanner_container_id=$(docker compose -p ${project_name} ${all_compose_files} ps -q wakufleet-scanner || true) if [[ -z "${scanner_container_id}" ]]; then echo -e "${RED}wakufleet-scanner service container not found. Make sure docker-compose.waku.yml defines it correctly.${RST}" exit 1 fi scanner_exit_code=$(docker wait "${scanner_container_id}") if [[ "${scanner_exit_code}" -ne 0 ]]; then echo -e "${RED}wakufleet-scanner failed with exit code ${scanner_exit_code}. See logs below:${RST}" docker logs "${scanner_container_id}" || true echo -e "${RED}Aborting functional tests because wakufleetconfig.json was not generated successfully.${RST}" exit 1 fi echo -e "${GRN}wakufleet-scanner completed successfully${RST}" # Resolve peer (old) backend images for cross-version compatibility tests. peer_image_args=() if [[ "${FUNCTIONAL_TESTS_MARKER}" == "compatibility" ]]; then # Run compatibility tests serially. They share a single waku fleet, and under # pytest-xdist parallelism the fleet's peer-exchange hands each light client the # ENRs of other concurrent tests' (torn-down) backends, churning filter peer # selection and intermittently dropping messages (see #7513). Serial execution # removes the cross-test contamination. FUNCTIONAL_TESTS_PARALLEL=0 echo -e "${GRN}Preparing peer images for compatibility tests${RST}" resolved_peer_images=() if [[ -n "${PEER_IMAGES}" ]]; then for img in ${PEER_IMAGES}; do docker pull "${img}" || { echo -e "${RED}Failed to pull peer image ${img}${RST}"; exit 1; } resolved_peer_images+=("${img}") done else if [[ -z "${PEER_REFS}" ]]; then echo "${GRN}Fetching git tags to determine peer refs${RST}" git fetch --tags --quiet || true PEER_REFS="$(git tag --sort=-v:refname 2>/dev/null | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | awk -F. '!seen[$1"."$2]++' | head -2 | paste -sd' ' -)" fi if [[ -z "${PEER_REFS}" ]]; then echo -e "${RED}No peer refs available. Set PEER_REFS or PEER_IMAGES.${RST}" exit 1 fi echo "${GRN}Peer refs:${RST} ${PEER_REFS}" # TODO: drop worktree build once released backend images are published to Harbor. for ref in ${PEER_REFS}; do ref_slug="$(printf '%s' "${ref}" | tr -c 'A-Za-z0-9_.-' '-' | sed 's/^[-.]*//; s/[-.]*$//')" peer_image="statusgo-peer-${ref_slug}" if ! docker image inspect "${peer_image}" > /dev/null 2>&1; then echo -e "${GRN}Building peer backend ${ref} -> ${peer_image}${RST}" git rev-parse --verify --quiet "${ref}^{commit}" > /dev/null || git fetch --tags origin "${ref}" peer_worktree="$(mktemp -d)" git worktree add --force --detach "${peer_worktree}" "${ref}" if ! docker build "${peer_worktree}" --build-arg "build_tags=${FUNCTIONAL_TESTS_BUILD_TAGS}" --tag "${peer_image}"; then echo -e "${RED}Failed to build peer backend ${ref}${RST}" git worktree remove --force "${peer_worktree}" || true exit 1 fi git worktree remove --force "${peer_worktree}" fi resolved_peer_images+=("${peer_image}") done fi for img in ${resolved_peer_images[@]+"${resolved_peer_images[@]}"}; do peer_image_args+=(--peer-docker-image="${img}") done echo -e "${GRN}Peer images:${RST} ${resolved_peer_images[*]+${resolved_peer_images[*]}}" fi # Set up virtual environment venv_path="${root_path}/.venv" if [[ -d "${venv_path}" ]]; then echo -e "${GRN}Using existing virtual environment${RST}" else echo -e "${GRN}Creating new virtual environment${RST}" python3 -m venv "${venv_path}" fi source "${venv_path}/bin/activate" # Upgrade pip and install requirements echo -e "${GRN}Installing dependencies${RST}" pip install --upgrade pip pip install -r "${root_path}/requirements.txt" # Run functional tests echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)" pytest --reruns "${FUNCTIONAL_TESTS_RERUNS}" -m "${pytest_marker_expr}" -c "${root_path}/pytest.ini" -n "${FUNCTIONAL_TESTS_PARALLEL}" \ --dist load\ --log-cli-level="${FUNCTIONAL_TESTS_LOG_LEVEL}" \ --docker_project_name="${project_name}" \ --docker-image=${image_name} \ ${peer_image_args[@]+"${peer_image_args[@]}"} \ --codecov_dir="${binary_coverage_reports_path}" \ --logs-dir="${logs_path}" \ --junitxml="${test_results_path}/report.xml" \ ${root_path} exit_code=$? # Stop containers echo -e "${GRN}Stopping docker containers${RST}" docker compose -p ${project_name} ${all_compose_files} stop dump_compose_logs # Cleanup containers echo -e "${GRN}Removing docker containers${RST}" docker compose -p ${project_name} ${all_compose_files} down # Collect coverage reports echo -e "${GRN}Collecting code coverage reports${RST}" full_coverage_profile="${coverage_reports_path}/coverage.out" go tool covdata merge -i="${binary_coverage_reports_path}" -o="${merged_coverage_reports_path}" go tool covdata textfmt -i="${merged_coverage_reports_path}" -o="${full_coverage_profile}" convert_coverage_to_html "${full_coverage_profile}" "${coverage_reports_path}/coverage.html" # Upload reports to Codecov if [[ ${FUNCTIONAL_TESTS_REPORT_CODECOV} == 'true' ]]; then report_to_codecov "${test_results_path}/*.xml" "${full_coverage_profile}" "functional" fi echo -e "${GRN}Testing finished${RST}" exit $exit_code