diff --git a/.github/workflows/continuous-tests.yaml b/.github/workflows/continuous-tests.yaml new file mode 100644 index 0000000..11947dd --- /dev/null +++ b/.github/workflows/continuous-tests.yaml @@ -0,0 +1,96 @@ +name: Run Continuous Tests + + +on: + # push: + # branches: + # - master + # tags: + # - 'v*.*.*' + # paths-ignore: + # - '**/*.md' + # - '.gitignore' + # - 'docker/**' + # - '!docker/continuous-tests-job.yaml' + workflow_dispatch: + inputs: + branch: + description: Branch (master) + required: false + type: string + source: + description: Repository with tests (current) + required: false + type: string + nameprefix: + description: Runner name prefix (codex-continuous-tests) + required: false + type: string + namespace: + description: Runner namespace (default) + required: false + type: string + tests_target_duration: + description: Runner target duration (172800s=48h) + required: false + type: string + tests_filter: + description: Runner tests filter ("") + required: false + type: string + tests_cleanup: + description: Runner tests cleanup (true) + required: false + type: string + deployment_namespace: + description: Deployment namespace (codex-continuous-tests) + required: false + type: string + + +env: + BRANCH: ${{ github.ref_name }} + SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }} + NAMEPREFIX: codex-continuous-tests + NAMESPACE: default + DEPLOYMENT_NAMESPACE: codex-continuous-tests + TESTS_TARGET_DURATION: 172800 + TESTS_FILTER: "" + TESTS_CLEANUP: true + JOB_MANIFEST: docker/continuous-tests-job.yaml + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + KUBE_VERSION: v1.28.2 + + +jobs: + run_tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Variables + run: | + [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV" + [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}" >>"$GITHUB_ENV" + echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV + echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + - name: Kubectl - Install ${{ env.KUBE_VERSION }} + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBE_VERSION }} + + - name: Kubectl - Kubeconfig + run: | + mkdir -p "${HOME}"/.kube + echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config + - name: Kubectl - Create Job + run: | + envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f - diff --git a/.github/workflows/dist-tests.yaml b/.github/workflows/dist-tests.yaml index 1453f00..631742c 100644 --- a/.github/workflows/dist-tests.yaml +++ b/.github/workflows/dist-tests.yaml @@ -1,4 +1,4 @@ -name: Dist Tests +name: Run Dist Tests on: @@ -11,7 +11,7 @@ on: # - '**/*.md' # - '.gitignore' # - 'docker/**' - # - '!docker/job.yaml' + # - '!docker/dist-tests-job.yaml' workflow_dispatch: inputs: branch: @@ -42,9 +42,9 @@ env: NAMEPREFIX: codex-dist-tests NAMESPACE: default COMMAND: dotnet test Tests/CodexTests - JOB_MANIFEST: docker/job.yaml + JOB_MANIFEST: docker/dist-tests-job.yaml KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} - KUBE_VERSION: v1.26.1 + KUBE_VERSION: v1.28.2 jobs: diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index d5273dd..3fdd5de 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -121,10 +121,12 @@ namespace KubernetesWorkflow SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory)); } - protected void SetResourceLimits(int milliCPUs, ByteSize memory) - { - SetResourceLimits(new ContainerResourceSet(milliCPUs, memory)); - } + // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is + // called for if they have resource limits defined. + //protected void SetResourceLimits(int milliCPUs, ByteSize memory) + //{ + // SetResourceLimits(new ContainerResourceSet(milliCPUs, memory)); + //} protected void SetResourcesRequest(ContainerResourceSet requests) { diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 9ea59a0..2d80be2 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -26,7 +26,7 @@ namespace CodexPlugin protected override void Initialize(StartupConfig startupConfig) { SetResourcesRequest(milliCPUs: 100, memory: 100.MB()); - SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); + //SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); var config = startupConfig.Get(); diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index 596ea2b..0916bdf 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -9,7 +9,7 @@ namespace ContinuousTests.Tests { public class TwoClientTest : ContinuousTest { - private const string BytesStoredMetric = "codexRepostoreBytesUsed"; + private const string BytesStoredMetric = "codex_repostore_bytes_used"; public override int RequiredNumberOfNodes => 2; public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2); diff --git a/Tests/CodexContinuousTests/run.sh b/Tests/CodexContinuousTests/run.sh index afc130f..f9e34b5 100644 --- a/Tests/CodexContinuousTests/run.sh +++ b/Tests/CodexContinuousTests/run.sh @@ -1,7 +1,12 @@ dotnet run \ - --kube-config=/opt/kubeconfig.yaml \ - --codex-deployment=codex-deployment.json \ - --log-path=/var/log/codex-continuous-tests/logs \ - --keep=1 \ - --stop=10 \ - --target-duration=172800 # 48 hours + --project "${TESTS_PATH:-Tests/CodexContinuousTests}" \ + --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \ + --codex-deployment="${TESTS_CODEX_DEPLOYMENT:-codex-deployment.json}" \ + --filter="${TESTS_FILTER:-}" \ + --data-path="${TESTS_DATA_PATH:-data}" \ + --log-path="${LOGPATH:-/var/log/codex-continuous-tests}" \ + --full-container-logs=1 \ + --keep=${TESTS_KEEP:-1} \ + --stop=${TESTS_STOP:-10} \ + --target-duration=${TESTS_TARGET_DURATION:-172800} \ + --cleanup="${TESTS_CLEANUP:-false}" diff --git a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh index 5269f22..9c87cbf 100644 --- a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh @@ -1,20 +1,21 @@ dotnet run \ - --deploy-name=codex-continuous-test-deployment \ - --kube-config=/opt/kubeconfig.yaml \ - --kube-namespace=codex-continuous-tests \ - --deploy-file=codex-deployment.json \ - --nodes=5 \ - --validators=3 \ - --log-level=Trace \ - --storage-quota=2048 \ - --make-storage-available=1 \ - --storage-sell=1024 \ - --min-price=1024 \ - --max-collateral=1024 \ - --max-duration=3600000 \ - --block-ttl=180 \ - --block-mi=120 \ - --block-mn=10000 \ - --metrics-endpoints=1 \ - --metrics-scraper=1 \ - --check-connect=1 + --project "${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}" \ + --deploy-name="${DEPLOYMENT_NAME:-codex-continuous-test-deployment}" \ + --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \ + --kube-namespace="${DEPLOYMENT_NAMESPACE:-codex-continuous-tests}" \ + --deploy-file="${DEPLOYMENT_FILE:-codex-deployment.json}" \ + --nodes=${DEPLOYMENT_NODES:-5} \ + --validators=${DEPLOYMENT_VALIDATORS:-3} \ + --log-level="${CODEX_LOG_LEVEL:-Trace}" \ + --storage-quota=${CODEX_STORAGE_QUOTA:-2048} \ + --storage-sell=${CODEX_STORAGE_SELL:-1024} \ + --min-price=${CODEX_MIN_PRICE:-1024} \ + --max-collateral=${CODEX_MAX_COLLATERAL:-1024} \ + --max-duration=${CODEX_MAX_DURATION:-3600000} \ + --block-ttl=${CODEX_BLOCK_TTL:-180} \ + --block-mi=${CODEX_BLOCK_MI:-120} \ + --block-mn=${CODEX_BLOCK_MN:-10000} \ + --metrics-endpoints=${CODEX_METRICS:-1} \ + --metrics-scraper=${CODEX_METRICS:-1} \ + --check-connect=${DEPLOYMENT_CHECK_CONNECT:-1} \ + -y diff --git a/docker/continuous-tests-job.yaml b/docker/continuous-tests-job.yaml new file mode 100644 index 0000000..fe86898 --- /dev/null +++ b/docker/continuous-tests-job.yaml @@ -0,0 +1,67 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ${NAMEPREFIX}-${RUNID} + namespace: ${NAMESPACE} + labels: + name: ${NAMEPREFIX}-${RUNID} + runid: ${RUNID} +spec: + backoffLimit: 0 + template: + metadata: + name: ${NAMEPREFIX} + labels: + app: continuous-tests-runner + name: ${NAMEPREFIX}-${RUNID} + runid: ${RUNID} + spec: + priorityClassName: system-node-critical + nodeSelector: + doks.digitalocean.com/node-pool: "fixed-s-4vcpu-16gb-amd" + containers: + - name: ${NAMEPREFIX}-runner + image: codexstorage/cs-codex-dist-tests:latest + imagePullPolicy: Always + resources: + requests: + memory: "2Gi" + env: + - name: KUBECONFIG + value: "/opt/kubeconfig.yaml" + - name: LOGPATH + value: "/var/log/codex-continuous-tests" + - name: NAMESPACE + value: "${NAMESPACE}" + - name: BRANCH + value: "${BRANCH}" + - name: SOURCE + value: "${SOURCE}" + - name: RUNID + value: "${RUNID}" + - name: TESTID + value: "${TESTID}" + - name: DEPLOYMENT_NAMESPACE + value: "${DEPLOYMENT_NAMESPACE}" + - name: TESTS_TYPE + value: "continuous-tests" + - name: TESTS_TARGET_DURATION + value: "${TESTS_TARGET_DURATION}" + - name: TESTS_FILTER + value: "${TESTS_FILTER}" + - name: TESTS_CLEANUP + value: "${TESTS_CLEANUP}" + volumeMounts: + - name: kubeconfig + mountPath: /opt/kubeconfig.yaml + subPath: kubeconfig.yaml + - name: logs + mountPath: /var/log/codex-continuous-tests + restartPolicy: Never + volumes: + - name: kubeconfig + secret: + secretName: codex-dist-tests-app-kubeconfig + - name: logs + hostPath: + path: /var/log/codex-continuous-tests diff --git a/docker/job.yaml b/docker/dist-tests-job.yaml similarity index 85% rename from docker/job.yaml rename to docker/dist-tests-job.yaml index 35d806b..7c92375 100644 --- a/docker/job.yaml +++ b/docker/dist-tests-job.yaml @@ -22,17 +22,17 @@ spec: imagePullPolicy: Always env: - name: KUBECONFIG - value: /opt/kubeconfig.yaml + value: "/opt/kubeconfig.yaml" - name: LOGPATH - value: /var/log/codex-dist-tests + value: "/var/log/codex-dist-tests" - name: NAMESPACE - value: ${NAMESPACE} + value: "${NAMESPACE}" - name: BRANCH - value: ${BRANCH} + value: "${BRANCH}" - name: SOURCE - value: ${SOURCE} + value: "${SOURCE}" - name: RUNID - value: ${RUNID} + value: "${RUNID}" - name: TESTID value: "${TESTID}" volumeMounts: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 10bc1d1..8e023b9 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,19 +1,35 @@ #!/bin/bash -# Variables +# Common SOURCE="${SOURCE:-https://github.com/codex-storage/cs-codex-dist-tests.git}" BRANCH="${BRANCH:-master}" -FOLDER="${FOLDER:-/opt/dist-tests}" +FOLDER="${FOLDER:-/opt/cs-codex-dist-tests}" + +# Continuous Tests +DEPLOYMENT_CODEXNETDEPLOYER_PATH="${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}" +DEPLOYMENT_CODEXNETDEPLOYER_RUNNER="${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER:-deploy-continuous-testnet.sh}" +CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/CodexContinuousTests}" +CONTINUOUS_TESTS_RUNNER="${CONTINUOUS_TESTS_RUNNER:-run.sh}" -# Get tests -echo "Clone ${SOURCE}" +# Get code +echo "`date` - Clone ${SOURCE}" git clone -b "${BRANCH}" "${SOURCE}" "${FOLDER}" -[[ -n "${CONFIG}" ]] && { echo Link config "${CONFIG}"; ln --symbolic --force "${CONFIG}" "${FOLDER}/DistTestCore/Configuration.cs"; } -[[ "${CONFIG_SHOW}" == "true" ]] && { echo Show config "${CONFIG}"; cat "${FOLDER}/DistTestCore/Configuration.cs"; } +echo "`date` - Change folder to ${FOLDER}" cd "${FOLDER}" # Run -echo "Run tests on branch '`git branch --show-current`' ..." -exec "$@" +echo "Run tests from branch '`git branch --show-current` / `git rev-parse HEAD`'" +if [[ "${TESTS_TYPE}" == "continuous-tests" ]]; then + echo "`date` - Running Continuous Tests" + echo + echo "`date` - Running CodexNetDeployer" + bash "${DEPLOYMENT_CODEXNETDEPLOYER_PATH}"/"${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER}" + echo + echo "`date` - Running Tests" + bash "${CONTINUOUS_TESTS_FOLDER}"/"${CONTINUOUS_TESTS_RUNNER}" +else + echo "`date` - Running Dist Tests" + exec "$@" +fi