From 807e8b7b5fef869a9196947cbf4f74864da8c4e7 Mon Sep 17 00:00:00 2001
From: Veaceslav Doina <20563034+veaceslavdoina@users.noreply.github.com>
Date: Mon, 6 Nov 2023 15:06:47 +0200
Subject: [PATCH 1/2] Fix TwoClientTest metric name

https://github.com/codex-storage/nim-codex/pull/603
---
 Tests/CodexContinuousTests/Tests/TwoClientTest.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs
index 596ea2b..0916bdf 100644
--- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs
+++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs
@@ -9,7 +9,7 @@ namespace ContinuousTests.Tests
 {
     public class TwoClientTest : ContinuousTest
     {
-        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
+        private const string BytesStoredMetric = "codex_repostore_bytes_used";
 
         public override int RequiredNumberOfNodes => 2;
         public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);

From ead1ff7ec6e67531fc3af5093b1f520f639c05a8 Mon Sep 17 00:00:00 2001
From: Veaceslav Doina <20563034+veaceslavdoina@users.noreply.github.com>
Date: Mon, 6 Nov 2023 15:09:51 +0200
Subject: [PATCH 2/2] Automate Continuous Tests using GitHub Actions (#69)

---
 .github/workflows/continuous-tests.yaml       | 96 +++++++++++++++++++
 .github/workflows/dist-tests.yaml             |  8 +-
 Tests/CodexContinuousTests/run.sh             | 16 +++-
 .../deploy-continuous-testnet.sh              | 36 +++----
 docker/continuous-tests-job.yaml              | 67 +++++++++++++
 docker/{job.yaml => dist-tests-job.yaml}      | 12 +--
 docker/docker-entrypoint.sh                   | 32 +++++--
 7 files changed, 227 insertions(+), 40 deletions(-)
 create mode 100644 .github/workflows/continuous-tests.yaml
 create mode 100644 docker/continuous-tests-job.yaml
 rename docker/{job.yaml => dist-tests-job.yaml} (85%)

diff --git a/.github/workflows/continuous-tests.yaml b/.github/workflows/continuous-tests.yaml
new file mode 100644
index 0000000..11947dd
--- /dev/null
+++ b/.github/workflows/continuous-tests.yaml
@@ -0,0 +1,96 @@
+name: Run Continuous Tests
+
+
+on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/continuous-tests-job.yaml'
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch (master)
+        required: false
+        type: string
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner name prefix (codex-continuous-tests)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (172800s=48h)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup (true)
+        required: false
+        type: string
+      deployment_namespace:
+        description: Deployment namespace (codex-continuous-tests)
+        required: false
+        type: string
+
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: codex-continuous-tests
+  NAMESPACE: default
+  DEPLOYMENT_NAMESPACE: codex-continuous-tests
+  TESTS_TARGET_DURATION: 172800
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Variables
+        run: |
+          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
diff --git a/.github/workflows/dist-tests.yaml b/.github/workflows/dist-tests.yaml
index 1453f00..631742c 100644
--- a/.github/workflows/dist-tests.yaml
+++ b/.github/workflows/dist-tests.yaml
@@ -1,4 +1,4 @@
-name: Dist Tests
+name: Run Dist Tests
 
 
 on:
@@ -11,7 +11,7 @@ on:
   #     - '**/*.md'
   #     - '.gitignore'
   #     - 'docker/**'
-  #     - '!docker/job.yaml'
+  #     - '!docker/dist-tests-job.yaml'
   workflow_dispatch:
     inputs:
       branch:
@@ -42,9 +42,9 @@ env:
   NAMEPREFIX: codex-dist-tests
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/job.yaml
+  JOB_MANIFEST: docker/dist-tests-job.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.26.1
+  KUBE_VERSION: v1.28.2
 
 
 jobs:
diff --git a/Tests/CodexContinuousTests/run.sh b/Tests/CodexContinuousTests/run.sh
index 59536db..f9e34b5 100644
--- a/Tests/CodexContinuousTests/run.sh
+++ b/Tests/CodexContinuousTests/run.sh
@@ -1,6 +1,12 @@
 dotnet run \
-    --kube-config=/opt/kubeconfig.yaml \
-    --codex-deployment=codex-deployment.json \
-    --keep=1 \
-    --stop=10 \
-    --target-duration=172800 # 48 hours
+    --project "${TESTS_PATH:-Tests/CodexContinuousTests}" \
+    --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \
+    --codex-deployment="${TESTS_CODEX_DEPLOYMENT:-codex-deployment.json}" \
+    --filter="${TESTS_FILTER:-}" \
+    --data-path="${TESTS_DATA_PATH:-data}" \
+    --log-path="${LOGPATH:-/var/log/codex-continuous-tests}" \
+    --full-container-logs=1 \
+    --keep=${TESTS_KEEP:-1} \
+    --stop=${TESTS_STOP:-10} \
+    --target-duration=${TESTS_TARGET_DURATION:-172800} \
+    --cleanup="${TESTS_CLEANUP:-false}"
diff --git a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh
index 2fe63a5..e3be58c 100644
--- a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh
+++ b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh
@@ -1,18 +1,20 @@
 dotnet run \
-    --deploy-name=codex-continuous-test-deployment \
-    --kube-config=/opt/kubeconfig.yaml \
-    --kube-namespace=codex-continuous-tests \
-    --deploy-file=codex-deployment.json \
-    --nodes=5 \
-    --validators=3 \
-    --log-level=Trace \
-    --storage-quota=2048 \
-    --storage-sell=1024 \
-    --min-price=1024 \
-    --max-collateral=1024 \
-    --max-duration=3600000 \
-    --block-ttl=180 \
-    --block-mi=120 \
-    --block-mn=10000 \
-    --metrics=1 \
-    --check-connect=1
+    --project "${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}" \
+    --deploy-name="${DEPLOYMENT_NAME:-codex-continuous-test-deployment}" \
+    --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \
+    --kube-namespace="${DEPLOYMENT_NAMESPACE:-codex-continuous-tests}" \
+    --deploy-file="${DEPLOYMENT_FILE:-codex-deployment.json}" \
+    --nodes=${DEPLOYMENT_NODES:-5} \
+    --validators=${DEPLOYMENT_VALIDATORS:-3} \
+    --log-level="${CODEX_LOG_LEVEL:-Trace}" \
+    --storage-quota=${CODEX_STORAGE_QUOTA:-2048} \
+    --storage-sell=${CODEX_STORAGE_SELL:-1024} \
+    --min-price=${CODEX_MIN_PRICE:-1024} \
+    --max-collateral=${CODEX_MAX_COLLATERAL:-1024} \
+    --max-duration=${CODEX_MAX_DURATION:-3600000} \
+    --block-ttl=${CODEX_BLOCK_TTL:-180} \
+    --block-mi=${CODEX_BLOCK_MI:-120} \
+    --block-mn=${CODEX_BLOCK_MN:-10000} \
+    --metrics=${CODEX_METRICS:-1} \
+    --check-connect=${DEPLOYMENT_CHECK_CONNECT:-1} \
+    -y
diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml
new file mode 100644
index 0000000..fe86898
--- /dev/null
+++ b/docker/continuous-tests-job.yaml
@@ -0,0 +1,67 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAMEPREFIX}-${RUNID}
+  namespace: ${NAMESPACE}
+  labels:
+    name: ${NAMEPREFIX}-${RUNID}
+    runid: ${RUNID}
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: ${NAMEPREFIX}
+      labels:
+        app: continuous-tests-runner
+        name: ${NAMEPREFIX}-${RUNID}
+        runid: ${RUNID}
+    spec:
+      priorityClassName: system-node-critical
+      nodeSelector:
+        doks.digitalocean.com/node-pool: "fixed-s-4vcpu-16gb-amd"
+      containers:
+      - name: ${NAMEPREFIX}-runner
+        image: codexstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "2Gi"
+        env:
+        - name: KUBECONFIG
+          value: "/opt/kubeconfig.yaml"
+        - name: LOGPATH
+          value: "/var/log/codex-continuous-tests"
+        - name: NAMESPACE
+          value: "${NAMESPACE}"
+        - name: BRANCH
+          value: "${BRANCH}"
+        - name: SOURCE
+          value: "${SOURCE}"
+        - name: RUNID
+          value: "${RUNID}"
+        - name: TESTID
+          value: "${TESTID}"
+        - name: DEPLOYMENT_NAMESPACE
+          value: "${DEPLOYMENT_NAMESPACE}"
+        - name: TESTS_TYPE
+          value: "continuous-tests"
+        - name: TESTS_TARGET_DURATION
+          value: "${TESTS_TARGET_DURATION}"
+        - name: TESTS_FILTER
+          value: "${TESTS_FILTER}"
+        - name: TESTS_CLEANUP
+          value: "${TESTS_CLEANUP}"
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/codex-continuous-tests
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/codex-continuous-tests
diff --git a/docker/job.yaml b/docker/dist-tests-job.yaml
similarity index 85%
rename from docker/job.yaml
rename to docker/dist-tests-job.yaml
index 35d806b..7c92375 100644
--- a/docker/job.yaml
+++ b/docker/dist-tests-job.yaml
@@ -22,17 +22,17 @@ spec:
         imagePullPolicy: Always
         env:
         - name: KUBECONFIG
-          value: /opt/kubeconfig.yaml
+          value: "/opt/kubeconfig.yaml"
         - name: LOGPATH
-          value: /var/log/codex-dist-tests
+          value: "/var/log/codex-dist-tests"
         - name: NAMESPACE
-          value: ${NAMESPACE}
+          value: "${NAMESPACE}"
         - name: BRANCH
-          value: ${BRANCH}
+          value: "${BRANCH}"
         - name: SOURCE
-          value: ${SOURCE}
+          value: "${SOURCE}"
         - name: RUNID
-          value: ${RUNID}
+          value: "${RUNID}"
         - name: TESTID
           value: "${TESTID}"
         volumeMounts:
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index 10bc1d1..8e023b9 100644
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -1,19 +1,35 @@
 #!/bin/bash
 
-# Variables
+# Common
 SOURCE="${SOURCE:-https://github.com/codex-storage/cs-codex-dist-tests.git}"
 BRANCH="${BRANCH:-master}"
-FOLDER="${FOLDER:-/opt/dist-tests}"
+FOLDER="${FOLDER:-/opt/cs-codex-dist-tests}"
+
+# Continuous Tests
+DEPLOYMENT_CODEXNETDEPLOYER_PATH="${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}"
+DEPLOYMENT_CODEXNETDEPLOYER_RUNNER="${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER:-deploy-continuous-testnet.sh}"
+CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/CodexContinuousTests}"
+CONTINUOUS_TESTS_RUNNER="${CONTINUOUS_TESTS_RUNNER:-run.sh}"
 
 
-# Get tests
-echo "Clone ${SOURCE}"
+# Get code
+echo "`date` - Clone ${SOURCE}"
 git clone -b "${BRANCH}" "${SOURCE}" "${FOLDER}"
-[[ -n "${CONFIG}" ]] && { echo Link config "${CONFIG}"; ln --symbolic --force "${CONFIG}" "${FOLDER}/DistTestCore/Configuration.cs"; }
-[[ "${CONFIG_SHOW}" == "true" ]] && { echo Show config "${CONFIG}"; cat "${FOLDER}/DistTestCore/Configuration.cs"; }
+echo "`date` - Change folder to ${FOLDER}"
 cd "${FOLDER}"
 
 # Run
-echo "Run tests on branch '`git branch --show-current`' ..."
-exec "$@"
+echo "Run tests from branch '`git branch --show-current` / `git rev-parse HEAD`'"
 
+if [[ "${TESTS_TYPE}" == "continuous-tests" ]]; then
+  echo "`date` - Running Continuous Tests"
+  echo
+  echo "`date` - Running CodexNetDeployer"
+  bash "${DEPLOYMENT_CODEXNETDEPLOYER_PATH}"/"${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER}"
+  echo
+  echo "`date` - Running Tests"
+  bash "${CONTINUOUS_TESTS_FOLDER}"/"${CONTINUOUS_TESTS_RUNNER}"
+else
+  echo "`date` - Running Dist Tests"
+  exec "$@"
+fi