Merge branch 'master' into deployment-json-rework

# Conflicts:
#	Tests/CodexContinuousTests/run.sh
#	Tools/CodexNetDeployer/deploy-continuous-testnet.sh

.github/workflows/continuous-tests.yaml

@@ -0,0 +1,96 @@
+name: Run Continuous Tests
+
+
+on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/continuous-tests-job.yaml'
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch (master)
+        required: false
+        type: string
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner name prefix (codex-continuous-tests)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (172800s=48h)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup (true)
+        required: false
+        type: string
+      deployment_namespace:
+        description: Deployment namespace (codex-continuous-tests)
+        required: false
+        type: string
+
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: codex-continuous-tests
+  NAMESPACE: default
+  DEPLOYMENT_NAMESPACE: codex-continuous-tests
+  TESTS_TARGET_DURATION: 172800
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Variables
+        run: |
+          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/dist-tests.yaml

@@ -1,4 +1,4 @@
-name: Dist Tests
+name: Run Dist Tests
 
 
 on:
@@ -11,7 +11,7 @@ on:
   #     - '**/*.md'
   #     - '.gitignore'
   #     - 'docker/**'
-  #     - '!docker/job.yaml'
+  #     - '!docker/dist-tests-job.yaml'
   workflow_dispatch:
     inputs:
       branch:
@@ -42,9 +42,9 @@ env:
   NAMEPREFIX: codex-dist-tests
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/job.yaml
+  JOB_MANIFEST: docker/dist-tests-job.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.26.1
+  KUBE_VERSION: v1.28.2
 
 
 jobs:

Framework/KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -121,10 +121,12 @@ namespace KubernetesWorkflow
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
-        protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
-        {
+        // called for if they have resource limits defined.
-            SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        //protected void SetResourceLimits(int milliCPUs, ByteSize memory)
-        }
+        //{
+        //    SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        //}
 
         protected void SetResourcesRequest(ContainerResourceSet requests)
         {

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -26,7 +26,7 @@ namespace CodexPlugin
         protected override void Initialize(StartupConfig startupConfig)
         {
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
-            SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
+            //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 
             var config = startupConfig.Get<CodexStartupConfig>();
 

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -9,7 +9,7 @@ namespace ContinuousTests.Tests
 {
     public class TwoClientTest : ContinuousTest
     {
-        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
+        private const string BytesStoredMetric = "codex_repostore_bytes_used";
 
         public override int RequiredNumberOfNodes => 2;
         public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);

Tests/CodexContinuousTests/run.sh

@@ -1,7 +1,12 @@
 dotnet run \
-    --kube-config=/opt/kubeconfig.yaml \
+    --project "${TESTS_PATH:-Tests/CodexContinuousTests}" \
-    --codex-deployment=codex-deployment.json \
+    --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \
-    --log-path=/var/log/codex-continuous-tests/logs \
+    --codex-deployment="${TESTS_CODEX_DEPLOYMENT:-codex-deployment.json}" \
-    --keep=1 \
+    --filter="${TESTS_FILTER:-}" \
-    --stop=10 \
+    --data-path="${TESTS_DATA_PATH:-data}" \
-    --target-duration=172800 # 48 hours
+    --log-path="${LOGPATH:-/var/log/codex-continuous-tests}" \
+    --full-container-logs=1 \
+    --keep=${TESTS_KEEP:-1} \
+    --stop=${TESTS_STOP:-10} \
+    --target-duration=${TESTS_TARGET_DURATION:-172800} \
+    --cleanup="${TESTS_CLEANUP:-false}"

Tools/CodexNetDeployer/deploy-continuous-testnet.sh

@@ -1,20 +1,21 @@
 dotnet run \
-    --deploy-name=codex-continuous-test-deployment \
+    --project "${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}" \
-    --kube-config=/opt/kubeconfig.yaml \
+    --deploy-name="${DEPLOYMENT_NAME:-codex-continuous-test-deployment}" \
-    --kube-namespace=codex-continuous-tests \
+    --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \
-    --deploy-file=codex-deployment.json \
+    --kube-namespace="${DEPLOYMENT_NAMESPACE:-codex-continuous-tests}" \
-    --nodes=5 \
+    --deploy-file="${DEPLOYMENT_FILE:-codex-deployment.json}" \
-    --validators=3 \
+    --nodes=${DEPLOYMENT_NODES:-5} \
-    --log-level=Trace \
+    --validators=${DEPLOYMENT_VALIDATORS:-3} \
-    --storage-quota=2048 \
+    --log-level="${CODEX_LOG_LEVEL:-Trace}" \
-    --make-storage-available=1 \
+    --storage-quota=${CODEX_STORAGE_QUOTA:-2048} \
-    --storage-sell=1024 \
+    --storage-sell=${CODEX_STORAGE_SELL:-1024} \
-    --min-price=1024 \
+    --min-price=${CODEX_MIN_PRICE:-1024} \
-    --max-collateral=1024 \
+    --max-collateral=${CODEX_MAX_COLLATERAL:-1024} \
-    --max-duration=3600000 \
+    --max-duration=${CODEX_MAX_DURATION:-3600000} \
-    --block-ttl=180 \
+    --block-ttl=${CODEX_BLOCK_TTL:-180} \
-    --block-mi=120 \
+    --block-mi=${CODEX_BLOCK_MI:-120} \
-    --block-mn=10000 \
+    --block-mn=${CODEX_BLOCK_MN:-10000} \
-    --metrics-endpoints=1 \
+    --metrics-endpoints=${CODEX_METRICS:-1} \
-    --metrics-scraper=1 \
+    --metrics-scraper=${CODEX_METRICS:-1} \
-    --check-connect=1
+    --check-connect=${DEPLOYMENT_CHECK_CONNECT:-1} \
+    -y

docker/continuous-tests-job.yaml

@@ -0,0 +1,67 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAMEPREFIX}-${RUNID}
+  namespace: ${NAMESPACE}
+  labels:
+    name: ${NAMEPREFIX}-${RUNID}
+    runid: ${RUNID}
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: ${NAMEPREFIX}
+      labels:
+        app: continuous-tests-runner
+        name: ${NAMEPREFIX}-${RUNID}
+        runid: ${RUNID}
+    spec:
+      priorityClassName: system-node-critical
+      nodeSelector:
+        doks.digitalocean.com/node-pool: "fixed-s-4vcpu-16gb-amd"
+      containers:
+      - name: ${NAMEPREFIX}-runner
+        image: codexstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "2Gi"
+        env:
+        - name: KUBECONFIG
+          value: "/opt/kubeconfig.yaml"
+        - name: LOGPATH
+          value: "/var/log/codex-continuous-tests"
+        - name: NAMESPACE
+          value: "${NAMESPACE}"
+        - name: BRANCH
+          value: "${BRANCH}"
+        - name: SOURCE
+          value: "${SOURCE}"
+        - name: RUNID
+          value: "${RUNID}"
+        - name: TESTID
+          value: "${TESTID}"
+        - name: DEPLOYMENT_NAMESPACE
+          value: "${DEPLOYMENT_NAMESPACE}"
+        - name: TESTS_TYPE
+          value: "continuous-tests"
+        - name: TESTS_TARGET_DURATION
+          value: "${TESTS_TARGET_DURATION}"
+        - name: TESTS_FILTER
+          value: "${TESTS_FILTER}"
+        - name: TESTS_CLEANUP
+          value: "${TESTS_CLEANUP}"
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/codex-continuous-tests
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/codex-continuous-tests

docker/dist-tests-job.yaml

@@ -22,17 +22,17 @@ spec:
         imagePullPolicy: Always
         env:
         - name: KUBECONFIG
-          value: /opt/kubeconfig.yaml
+          value: "/opt/kubeconfig.yaml"
         - name: LOGPATH
-          value: /var/log/codex-dist-tests
+          value: "/var/log/codex-dist-tests"
         - name: NAMESPACE
-          value: ${NAMESPACE}
+          value: "${NAMESPACE}"
         - name: BRANCH
-          value: ${BRANCH}
+          value: "${BRANCH}"
         - name: SOURCE
-          value: ${SOURCE}
+          value: "${SOURCE}"
         - name: RUNID
-          value: ${RUNID}
+          value: "${RUNID}"
         - name: TESTID
           value: "${TESTID}"
         volumeMounts:

docker/docker-entrypoint.sh

@@ -1,19 +1,35 @@
 #!/bin/bash
 
-# Variables
+# Common
 SOURCE="${SOURCE:-https://github.com/codex-storage/cs-codex-dist-tests.git}"
 BRANCH="${BRANCH:-master}"
-FOLDER="${FOLDER:-/opt/dist-tests}"
+FOLDER="${FOLDER:-/opt/cs-codex-dist-tests}"
+
+# Continuous Tests
+DEPLOYMENT_CODEXNETDEPLOYER_PATH="${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}"
+DEPLOYMENT_CODEXNETDEPLOYER_RUNNER="${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER:-deploy-continuous-testnet.sh}"
+CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/CodexContinuousTests}"
+CONTINUOUS_TESTS_RUNNER="${CONTINUOUS_TESTS_RUNNER:-run.sh}"
 
 
-# Get tests
+# Get code
-echo "Clone ${SOURCE}"
+echo "`date` - Clone ${SOURCE}"
 git clone -b "${BRANCH}" "${SOURCE}" "${FOLDER}"
-[[ -n "${CONFIG}" ]] && { echo Link config "${CONFIG}"; ln --symbolic --force "${CONFIG}" "${FOLDER}/DistTestCore/Configuration.cs"; }
+echo "`date` - Change folder to ${FOLDER}"
-[[ "${CONFIG_SHOW}" == "true" ]] && { echo Show config "${CONFIG}"; cat "${FOLDER}/DistTestCore/Configuration.cs"; }
 cd "${FOLDER}"
 
 # Run
-echo "Run tests on branch '`git branch --show-current`' ..."
+echo "Run tests from branch '`git branch --show-current` / `git rev-parse HEAD`'"
-exec "$@"
 
+if [[ "${TESTS_TYPE}" == "continuous-tests" ]]; then
+  echo "`date` - Running Continuous Tests"
+  echo
+  echo "`date` - Running CodexNetDeployer"
+  bash "${DEPLOYMENT_CODEXNETDEPLOYER_PATH}"/"${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER}"
+  echo
+  echo "`date` - Running Tests"
+  bash "${CONTINUOUS_TESTS_FOLDER}"/"${CONTINUOUS_TESTS_RUNNER}"
+else
+  echo "`date` - Running Dist Tests"
+  exec "$@"
+fi