status-go/_assets/scripts/test_stats.py

#!/usr/bin/env python

import glob
import xml.etree.ElementTree as ET
from collections import defaultdict
import re

test_stats = defaultdict(lambda: defaultdict(int))
skipped_tests = {}  # Use a dictionary to store test names and their skip reasons

file_path = "**/report_*.xml"

for file in glob.glob(file_path, recursive=True):
    tree = ET.parse(file)
    root = tree.getroot()
    for testcase in root.iter("testcase"):
        test_name = testcase.attrib["name"]

        test_stats[test_name]["total_runs"] += 1

        if testcase.find("failure") is not None:
            test_stats[test_name]["failed_runs"] += 1
        elif testcase.find("error") is not None:
            test_stats[test_name]["failed_runs"] += 1

        # Check for skipped tests
        skipped_element = testcase.find("skipped")
        if skipped_element is not None:
            message = skipped_element.attrib.get("message", "")
            # Extract the real reason from the message
            match = re.search(r': (.*?)\s*--- SKIP', message)
            skip_reason = match.group(1).strip() if match else "unknown reason"
            skipped_tests[test_name] = skip_reason  # Store test name and skip reason

# Filter out root test cases if they have subtests
filtered_test_stats = {
    name: stats for name, stats in test_stats.items()
    if not any(subtest.startswith(name + "/") for subtest in test_stats)
}

failing_test_stats = [
    {
        "name": name,
        "failure_rate": stats["failed_runs"] / stats["total_runs"],
        "failed_runs": stats["failed_runs"],
        "total_runs": stats["total_runs"]
    }
    for name, stats in filtered_test_stats.items() if stats["failed_runs"] != 0
]

sorted_failing_test_stats = sorted(failing_test_stats,
                                   key=lambda x: x["failure_rate"],
                                   reverse=True)

flaky_skipped_count = sum(1 for reason in skipped_tests.values() if reason == "flaky test")

print("---")
print(f"Failing tests stats (total: {len(failing_test_stats)})")
print("---")
for test_stat in sorted_failing_test_stats:
    print("{}: {:.1f}% ({} of {} failed)".format(
        test_stat['name'],
        test_stat['failure_rate'] * 100,
        test_stat['failed_runs'],
        test_stat['total_runs']
    ))

print("---")
print(f"Skipped tests (total: {len(skipped_tests)}, skipped as flaky: {flaky_skipped_count})")
print("---")
for test_name, skip_reason in skipped_tests.items():
    print(f"{test_name}: {skip_reason}")