mirror of
https://github.com/logos-messaging/logos-messaging-interop-tests.git
synced 2026-01-02 22:13:11 +00:00
ad74b21392
* test: overall job result * test: overall job result with nim daily * test: overall job result with nim daily success case * test: overall job result with nim daily failure case * test: replace deprecated set-output with GITHUB_OUTPUT * test: change OUTCOMES evaluation * test: debug OUTCOMES evaluation * test: print OUTCOMES * test: print OUTCOMES remove star * test: print outputs * test: print GITHUB_OUTPUT * test: print JOB_RESULT * test: print JOB_RESULT in reports * test: print ALL_OUTPUTS * test: print ALL_OUTPUTS 2 * test: print ALL_OUTPUTS 3 * test: print ALL_OUTPUTS with curly * test: print OUTCOMES * test: print jobResult * test: print all jobResults * fix: shard as version input * fix: process results * test: full workflow - failure case * test: full workflow - failure case - manual * test: full workflow - success case - RLN tests re-enabled * fix: make workflows ready to merge - disable RLN tests again
196 lines
7.4 KiB
YAML
196 lines
7.4 KiB
YAML
name: Interop Tests Common
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
node1:
|
|
required: true
|
|
description: "Node that usually publishes messages. Used for all tests"
|
|
type: string
|
|
default: "wakuorg/go-waku:latest"
|
|
node2:
|
|
required: true
|
|
description: "Node that usually queries for published messages. Used for all tests"
|
|
type: string
|
|
default: "wakuorg/nwaku:latest"
|
|
additional_nodes:
|
|
required: false
|
|
description: "Additional optional nodes used in e2e tests, separated by ,"
|
|
type: string
|
|
default: "wakuorg/nwaku:latest,wakuorg/go-waku:latest,wakuorg/nwaku:latest"
|
|
caller:
|
|
required: false
|
|
description: "Workflow caller. Used in reporting"
|
|
type: string
|
|
|
|
env:
|
|
FORCE_COLOR: "1"
|
|
NODE_1: ${{ inputs.node1 }}
|
|
NODE_2: ${{ inputs.node2 }}
|
|
ADDITIONAL_NODES: ${{ inputs.additional_nodes }}
|
|
CALLER: ${{ inputs.caller || 'manual' }}
|
|
RLN_CREDENTIALS: ${{ secrets.RLN_CREDENTIALS }}
|
|
|
|
jobs:
|
|
|
|
tests:
|
|
name: tests
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]
|
|
# total number of shards =18 means tests will split into 18 thread and run in parallel to increase execution speed
|
|
# command for sharding :
|
|
# pytest --shard-id=<shard_number> --num-shards=<total_shards>
|
|
# shard 16 for test_rln.py file as they shall run sequentially
|
|
# shard 17 for test_cursor_many_msgs.py as it takes time >7 mins
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 120
|
|
outputs:
|
|
jobResult_0: ${{ steps.set_result.outputs.JOB_RESULT_0 }}
|
|
jobResult_1: ${{ steps.set_result.outputs.JOB_RESULT_1 }}
|
|
jobResult_2: ${{ steps.set_result.outputs.JOB_RESULT_2 }}
|
|
jobResult_3: ${{ steps.set_result.outputs.JOB_RESULT_3 }}
|
|
jobResult_4: ${{ steps.set_result.outputs.JOB_RESULT_4 }}
|
|
jobResult_5: ${{ steps.set_result.outputs.JOB_RESULT_5 }}
|
|
jobResult_6: ${{ steps.set_result.outputs.JOB_RESULT_6 }}
|
|
jobResult_7: ${{ steps.set_result.outputs.JOB_RESULT_7 }}
|
|
jobResult_8: ${{ steps.set_result.outputs.JOB_RESULT_8 }}
|
|
jobResult_9: ${{ steps.set_result.outputs.JOB_RESULT_9 }}
|
|
jobResult_10: ${{ steps.set_result.outputs.JOB_RESULT_10 }}
|
|
jobResult_11: ${{ steps.set_result.outputs.JOB_RESULT_11 }}
|
|
jobResult_12: ${{ steps.set_result.outputs.JOB_RESULT_12 }}
|
|
jobResult_13: ${{ steps.set_result.outputs.JOB_RESULT_13 }}
|
|
jobResult_14: ${{ steps.set_result.outputs.JOB_RESULT_14 }}
|
|
jobResult_15: ${{ steps.set_result.outputs.JOB_RESULT_15 }}
|
|
jobResult_16: ${{ steps.set_result.outputs.JOB_RESULT_16 }}
|
|
jobResult_17: ${{ steps.set_result.outputs.JOB_RESULT_17 }}
|
|
steps:
|
|
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Remove unwanted software
|
|
uses: ./.github/actions/prune-vm
|
|
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.12'
|
|
cache: 'pip'
|
|
|
|
- run: pip install -r requirements.txt
|
|
|
|
- name: Run tests
|
|
|
|
run: |
|
|
if [ "${{ matrix.shard }}" == "16" ]; then
|
|
pytest tests/relay/test_rln.py --alluredir=allure-results-${{ matrix.shard }}
|
|
elif [ "${{ matrix.shard }}" == "17" ]; then
|
|
pytest tests/store/test_cursor_many_msgs.py --alluredir=allure-results-${{ matrix.shard }}
|
|
elif [ "${{ matrix.shard }}" != "17" ]; then
|
|
pytest --ignore=tests/relay/test_rln.py --ignore=tests/store/test_cursor_many_msgs.py --reruns 2 --shard-id=${{ matrix.shard }} --num-shards=16 --alluredir=allure-results-${{ matrix.shard }}
|
|
fi
|
|
|
|
- name: Upload allure results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: allure-results-${{ matrix.shard }}
|
|
path: allure-results-${{ matrix.shard }}
|
|
|
|
- name: Set job result
|
|
id: set_result
|
|
if: always()
|
|
run: |
|
|
version="${{ matrix.shard }}"
|
|
echo "JOB_RESULT_${version}=${{ job.status }}" >> "$GITHUB_OUTPUT"
|
|
|
|
|
|
aggregate-reports:
|
|
runs-on: ubuntu-latest
|
|
needs: [tests]
|
|
if: always()
|
|
steps:
|
|
- name: Download all allure results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: all-results
|
|
merge-multiple: true
|
|
|
|
- name: Get allure history
|
|
if: always()
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: gh-pages
|
|
path: gh-pages
|
|
|
|
- name: Setup allure report
|
|
uses: simple-elf/allure-report-action@master
|
|
if: always()
|
|
id: allure-report
|
|
with:
|
|
allure_results: all-results
|
|
gh_pages: gh-pages/${{ env.CALLER }}
|
|
allure_history: allure-history
|
|
keep_reports: 30
|
|
report_url: https://waku-org.github.io/waku-interop-tests/${{ env.CALLER }}
|
|
|
|
- name: Deploy report to Github Pages
|
|
uses: peaceiris/actions-gh-pages@v3
|
|
if: always()
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
publish_branch: gh-pages
|
|
publish_dir: allure-history
|
|
destination_dir: ${{ env.CALLER }}
|
|
|
|
- name: Store output from matrix jobs
|
|
run: |
|
|
echo '${{ toJSON(needs.tests.outputs) }}' > results.json
|
|
|
|
- name: Create job summary
|
|
if: always()
|
|
run: |
|
|
echo "## Run Information" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Event**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Actor**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Node1**: ${{ env.NODE_1 }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Node2**: ${{ env.NODE_2 }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Additonal Nodes**: ${{ env.ADDITIONAL_NODES }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "## Test Results" >> $GITHUB_STEP_SUMMARY
|
|
echo "Allure report will be available at: https://waku-org.github.io/waku-interop-tests/${{ env.CALLER }}/${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY
|
|
|
|
# Evaluate overall result
|
|
TESTS_RESULT="success"
|
|
for key in $(jq -r 'keys[]' results.json); do
|
|
result=$(jq -r --arg key "$key" '.[$key]' results.json)
|
|
echo "Key: $key, Value: $result"
|
|
# Check condition on the result
|
|
if [ "$result" != "success" ]; then
|
|
echo "Value 'success' not found at key: $key"
|
|
TESTS_RESULT="failure"
|
|
break
|
|
fi
|
|
done
|
|
|
|
# Notify Waku team
|
|
if [ "$TESTS_RESULT" != "success" ]; then
|
|
echo "There are failures with nwaku node. cc <@&1111608257824440330>" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
# Write result and summary to ENV
|
|
echo "TESTS_RESULT=$TESTS_RESULT" >> $GITHUB_ENV
|
|
{
|
|
echo 'JOB_SUMMARY<<EOF'
|
|
cat $GITHUB_STEP_SUMMARY
|
|
echo EOF
|
|
} >> $GITHUB_ENV
|
|
|
|
- name: Send report to Discord
|
|
uses: rjstone/discord-webhook-notify@v1
|
|
if: always() && env.CALLER != 'manual'
|
|
with:
|
|
severity: ${{ env.TESTS_RESULT == 'success' && 'info' || 'error' }}
|
|
username: ${{ github.workflow }}
|
|
description: "## Job Result: ${{ env.TESTS_RESULT }}"
|
|
details: ${{ env.JOB_SUMMARY }}
|
|
webhookUrl: ${{ secrets.DISCORD_TEST_REPORTS_WH }} |