From 6f53e78e6a256f18c9608209b007f3053c7fd649 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Sun, 19 Nov 2023 21:13:40 +0200 Subject: [PATCH] Run C-Tests reports by Scheduler (#80) --- .github/workflows/continuous-tests.yaml | 102 ++++++++++++------- .github/workflows/dist-tests.yaml | 24 ++--- .github/workflows/report-HoldMyBeerTest.yaml | 20 ++++ .github/workflows/report-PeersTest.yaml | 20 ++++ docker/continuous-tests-job.yaml | 6 +- 5 files changed, 117 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/report-HoldMyBeerTest.yaml create mode 100644 .github/workflows/report-PeersTest.yaml diff --git a/.github/workflows/continuous-tests.yaml b/.github/workflows/continuous-tests.yaml index be4d2588..6eab3279 100644 --- a/.github/workflows/continuous-tests.yaml +++ b/.github/workflows/continuous-tests.yaml @@ -2,16 +2,6 @@ name: Run Continuous Tests on: - # push: - # branches: - # - master - # tags: - # - 'v*.*.*' - # paths-ignore: - # - '**/*.md' - # - '.gitignore' - # - 'docker/**' - # - '!docker/continuous-tests-job.yaml' workflow_dispatch: inputs: source: @@ -23,15 +13,11 @@ on: required: false type: string nameprefix: - description: Runner name prefix (c-tests-runner) - required: false - type: string - namespace: - description: Runner namespace (default) + description: Resources prefix (c-tests) required: false type: string tests_target_duration: - description: Runner target duration (172800 = 48h) + description: Runner target duration (2d) required: false type: string tests_filter: @@ -40,24 +26,42 @@ on: type: string tests_cleanup: description: Runner tests cleanup - type: choice - options: - - true - - false + type: boolean default: true - deployment_namespace: - description: Deployment namespace (c-tests-$runid) + workflow_call: + inputs: + source: + description: Repository with tests (current) required: false type: string + branch: + description: Branch with tests (master) + required: false + type: string + nameprefix: + description: Resources prefix (c-tests) + required: false + type: string + tests_target_duration: + description: Runner target duration (2d) + required: false + type: string + tests_filter: + description: Runner tests filter ("") + required: false + type: string + tests_cleanup: + description: Runner tests cleanup + required: false + type: boolean env: - BRANCH: ${{ github.ref_name }} SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }} - NAMEPREFIX: c-tests-runner + BRANCH: ${{ github.ref_name }} + NAMEPREFIX: c-tests NAMESPACE: default - DEPLOYMENT_NAMESPACE: c-tests - TESTS_TARGET_DURATION: 172800 + TESTS_TARGET_DURATION: 2d TESTS_FILTER: "" TESTS_CLEANUP: true JOB_MANIFEST: docker/continuous-tests-job.yaml @@ -78,14 +82,14 @@ jobs: RUNID=$(date +%Y%m%d-%H%M%S) echo "RUNID=${RUNID}" >> $GITHUB_ENV echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}-${RUNID}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.nameprefix }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ inputs.nameprefix }}-${RUNID}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.NAMEPREFIX }}-${RUNID}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV" - name: Kubectl - Install ${{ env.KUBE_VERSION }} uses: azure/setup-kubectl@v3 @@ -106,7 +110,8 @@ jobs: echo "----" echo "Repository: ${{ env.SOURCE }}" echo "Branch: ${{ env.BRANCH }}" - echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}" + echo "Runner job: ${{ env.NAMEPREFIX }}" + echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`" echo "Runner namespace: ${{ env.NAMESPACE }}" echo "----" echo "Tests runid: ${{ env.RUNID }}" @@ -115,3 +120,30 @@ jobs: echo "Tests filter: ${{ env.TESTS_FILTER }}" echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}" echo "----" + + - name: Show Runner logs + if: false + run: | + # Variables + # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node + duration=600 + namespace="${{ env.NAMESPACE }}" + pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}") + + # Check Pod status + WAIT=120 + SECONDS=0 + sleep=1 + while (( SECONDS < WAIT )); do + phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}") + [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; } + sleep $sleep + done + + # Get logs + timeout $duration \ + kubectl logs $pod \ + -n $namespace \ + -f \ + --tail=-1 \ + --timestamps || true diff --git a/.github/workflows/dist-tests.yaml b/.github/workflows/dist-tests.yaml index a46943e3..d629b9df 100644 --- a/.github/workflows/dist-tests.yaml +++ b/.github/workflows/dist-tests.yaml @@ -2,16 +2,6 @@ name: Run Dist Tests on: - # push: - # branches: - # - master - # tags: - # - 'v*.*.*' - # paths-ignore: - # - '**/*.md' - # - '.gitignore' - # - 'docker/**' - # - '!docker/dist-tests-job.yaml' workflow_dispatch: inputs: source: @@ -37,8 +27,8 @@ on: env: - BRANCH: ${{ github.ref_name }} SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }} + BRANCH: ${{ github.ref_name }} NAMEPREFIX: d-tests-runner NAMESPACE: default COMMAND: dotnet test Tests/CodexTests @@ -57,14 +47,14 @@ jobs: - name: Variables run: | - [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" - [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}" echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV" echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" + [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}" - name: Kubectl - Install ${{ env.KUBE_VERSION }} uses: azure/setup-kubectl@v3 @@ -85,7 +75,7 @@ jobs: echo "----" echo "Repository: ${{ env.SOURCE }}" echo "Branch: ${{ env.BRANCH }}" - echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}" + echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}" echo "Runner namespace: ${{ env.NAMESPACE }}" echo "----" echo "Tests runid: ${{ env.RUNID }}" diff --git a/.github/workflows/report-HoldMyBeerTest.yaml b/.github/workflows/report-HoldMyBeerTest.yaml new file mode 100644 index 00000000..7bd78904 --- /dev/null +++ b/.github/workflows/report-HoldMyBeerTest.yaml @@ -0,0 +1,20 @@ +name: Report - HoldMyBeerTest + + +on: + schedule: + - cron: '30 */49 * * *' + workflow_dispatch: + +jobs: + run_tests: + name: Call runner + uses: ./.github/workflows/continuous-tests.yaml + with: + source: ${{ format('{0}/{1}', github.server_url, github.repository) }} + branch: master + nameprefix: c-tests-report-holdmybeertest + tests_target_duration: 48h + tests_filter: HoldMyBeerTest + tests_cleanup: true + secrets: inherit diff --git a/.github/workflows/report-PeersTest.yaml b/.github/workflows/report-PeersTest.yaml new file mode 100644 index 00000000..0f1166bc --- /dev/null +++ b/.github/workflows/report-PeersTest.yaml @@ -0,0 +1,20 @@ +name: Report - PeersTest + + +on: + schedule: + - cron: '30 */49 * * *' + workflow_dispatch: + +jobs: + run_tests: + name: Call runner + uses: ./.github/workflows/continuous-tests.yaml + with: + source: ${{ format('{0}/{1}', github.server_url, github.repository) }} + branch: master + nameprefix: c-tests-report-peerstest + tests_target_duration: 48h + tests_filter: PeersTest + tests_cleanup: true + secrets: inherit diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml index c669ec04..f4399812 100644 --- a/docker/continuous-tests-job.yaml +++ b/docker/continuous-tests-job.yaml @@ -1,10 +1,10 @@ apiVersion: batch/v1 kind: Job metadata: - name: ${NAMEPREFIX}-${RUNID} + name: ${NAMEPREFIX} namespace: ${NAMESPACE} labels: - name: ${NAMEPREFIX}-${RUNID} + name: ${NAMEPREFIX} runid: ${RUNID} spec: backoffLimit: 0 @@ -13,7 +13,7 @@ spec: name: ${NAMEPREFIX} labels: app: continuous-tests-runner - name: ${NAMEPREFIX}-${RUNID} + name: ${NAMEPREFIX} runid: ${RUNID} spec: priorityClassName: system-node-critical