test_: Test reliability workflow (#6236)

* test_: add test reliability workflow

* test_: run docker detached

* test_: run inside tests functional

* test_: fix pytest config

* test_: fix test

* test_: add allure reporting

* test_: add allure reporting

* test_: add allure reporting

* test_: use PAT

* test_: use PAT

* test_: test pat

* test_: always create reports

* test_: use dedicated repo

* test_: remove unneeded step

* test_: remove on push

* test_: add nightly job

.github/workflows/test-reliability.yml

@@ -0,0 +1,83 @@
+name: Test Reliability
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:  '0 2 * * *'
+
+env:
+  FORCE_COLOR: "1"
+  DEPLOY_TEST_REPORTS_PAT: ${{ secrets.DEPLOY_TEST_REPORTS_PAT }}
+
+jobs:
+  test-reliability:
+    timeout-minutes: 90
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+        cache: 'pip'
+
+    - name: Set up virtual environment in /tests-functional/
+      run: |
+        python -m venv tests-functional/.venv
+        echo "tests-functional/.venv/bin" >> $GITHUB_PATH  # Add virtualenv to PATH for subsequent steps
+
+    - name: Install dependencies based on requirements.txt
+      run: pip install -r tests-functional/requirements.txt
+
+    - name: Build status-backend
+      run: cd tests-functional; docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans -d
+
+    - name: Run tests
+      run: pytest -m "reliability" -c tests-functional/pytest.ini --alluredir=allure-results
+
+    - name: Get allure history
+      if: always()
+      uses: actions/checkout@v4
+      continue-on-error: true
+      with:
+        repository: status-im/status-go-allure
+        ref: gh-pages
+        path: gh-pages
+        token: ${{ env.DEPLOY_TEST_REPORTS_PAT }}
+
+    - name: Setup allure report
+      uses: simple-elf/allure-report-action@master
+      if: always()
+      id: allure-report
+      with:
+        allure_results: allure-results
+        gh_pages: gh-pages
+        allure_history: allure-history
+        keep_reports: 30
+        report_url: 'https://status-im.github.io/status-go-allure/'
+
+    - name: Deploy report to Github Pages
+      uses: peaceiris/actions-gh-pages@v4
+      if: always()
+      with:
+        personal_token: ${{ env.DEPLOY_TEST_REPORTS_PAT }}
+        external_repository: status-im/status-go-allure
+        publish_branch: gh-pages
+        publish_dir: allure-history
+
+    - name: Create job summary
+      if: always()
+      env:
+        JOB_STATUS: ${{ job.status }}
+      run: |
+        echo "## Run Information" >> $GITHUB_STEP_SUMMARY
+        echo "- **Event**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Actor**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
+        echo "## Test Results" >> $GITHUB_STEP_SUMMARY
+        echo "Allure report will be available at: https://status-im.github.io/status-go-allure/${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY
+        {
+          echo 'JOB_SUMMARY<<EOF'
+          cat $GITHUB_STEP_SUMMARY
+          echo EOF
+        } >> $GITHUB_ENV

tests-functional/pytest.ini

@@ -14,3 +14,4 @@ markers =
     init
     transaction
     create_account
+    reliability

tests-functional/requirements.txt

@@ -10,3 +10,5 @@ docker==7.1.0
 pyright==1.1.388
 black==24.10.0
 pre-commit==3.6.2
+allure-pytest==2.13.5
+allure-python-commons==2.13.5

tests-functional/tests/reliability/test_contact_request.py

@@ -82,12 +82,8 @@ class TestContactRequests(MessengerTestCase):
 
     @pytest.mark.dependency(depends=["test_contact_request_baseline"])
     def test_contact_request_with_node_pause_30_seconds(self):
-        await_signals = [
+        sender = self.initialize_backend(await_signals=self.await_signals)
-            SignalType.MESSAGES_NEW.value,
+        receiver = self.initialize_backend(await_signals=self.await_signals)
-            SignalType.MESSAGE_DELIVERED.value,
-        ]
-        sender = self.initialize_backend(await_signals=await_signals)
-        receiver = self.initialize_backend(await_signals=await_signals)
 
         with self.node_pause(receiver):
             message_text = f"test_contact_request_{uuid4()}"