Merge branch 'master' into plugin-refactor

.github/workflows/dist-tests.yaml

@@ -23,11 +23,15 @@ on:
         required: false
         type: string
       nameprefix:
-        description: Runner prefix (cs-codex-dist-tests)
+        description: Runner prefix (codex-dist-tests)
         required: false
         type: string
       namespace:
-        description: Runner namespace (cs-codex-dist-tests)
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Runner command (dotnet test Tests)
         required: false
         type: string
 
@@ -35,8 +39,9 @@ on:
 env:
   BRANCH: ${{ github.ref_name }}
   SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: cs-codex-dist-tests
+  NAMEPREFIX: codex-dist-tests
-  NAMESPACE: cs-codex-dist-tests
+  NAMESPACE: default
+  COMMAND: dotnet test Tests
   JOB_MANIFEST: docker/job.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.26.1
@@ -56,6 +61,8 @@ jobs:
           [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
           echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
           echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 

docker/job.yaml

@@ -11,18 +11,20 @@ spec:
   template:
     metadata:
       name: ${NAMEPREFIX}
+      labels:
+        app: dist-tests-runner
+        name: ${NAMEPREFIX}-${RUNID}
+        run-id: ${RUNID}
     spec:
       containers:
       - name: ${NAMEPREFIX}-runner
         image: codexstorage/cs-codex-dist-tests:latest
         imagePullPolicy: Always
         env:
-        - name: RUNNERLOCATION
-          value: InternalToCluster
         - name: KUBECONFIG
           value: /opt/kubeconfig.yaml
         - name: LOGPATH
-          value: /var/log/cs-codex-dist-tests
+          value: /var/log/codex-dist-tests
         - name: NAMESPACE
           value: ${NAMESPACE}
         - name: BRANCH
@@ -32,22 +34,19 @@ spec:
         - name: RUNID
           value: ${RUNID}
         - name: TESTID
-          value: ${TESTID}
+          value: "${TESTID}"
         volumeMounts:
         - name: kubeconfig
           mountPath: /opt/kubeconfig.yaml
           subPath: kubeconfig.yaml
         - name: logs
-          mountPath: /var/log/cs-codex-dist-tests
+          mountPath: /var/log/codex-dist-tests
-        # command:
+        args: ${COMMAND}
-        # - "dotnet"
-        # - "test"
-        # - "Tests"
       restartPolicy: Never
       volumes:
         - name: kubeconfig
           secret:
-            secretName: cs-codex-dist-tests-app-kubeconfig
+            secretName: codex-dist-tests-app-kubeconfig
         - name: logs
           hostPath:
-            path: /var/log/cs-codex-dist-tests
+            path: /var/log/codex-dist-tests

docs/Automation.md

@@ -1,4 +1,4 @@
-# Distributed Tests automation
+# Tests automation
 
  1. [Description](#description)
  2. [Architecture](#architecture)
@@ -15,7 +15,7 @@
 
  We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster.
 
- Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](/ContinuousTests) which are called to detect issues on continuous Codex runs.
+ Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](Continuous-Tests.md) which are called to detect issues on continuous Codex runs.
 
 
 ## Architecture
@@ -23,12 +23,20 @@
 <img src="Architecture.png" alt="Architecture" width="800"/>
 
 ```
-                                        Logs     --> Kibana
+                                        Vector --> Elaticsearch --> Kibana
-                                      /                |
+                                      / (Logs)         |
-GitHub --> CI --> Kubernetes -->  Job    Prometheus  Elaticsearch
+GitHub --> CI --> Kubernetes -->  Job                  |
-      \                          /    \  /         \   |
+      \             /                 \                |
-        ------------------------        Metrics  --> Grafana
+        -----------                     Prometheus --> Grafana
+                                        (Metrics)
 ```
+ 1. GitHub Actions run a workflow
+ 2. This workflow create a Job in Kubernetes cluster
+ 3. Job run Dist-Tests runner Pod with specified parameters
+ 4. Dists-Tests runner run the tests from inside the Kubernetes and generate the logs
+ 5. Vector ship the logs of the Dists-Tests Pods
+ 6. Prometheus collect the metrics of the Dists-Tests Codex Pods
+ 7. We can see the status of the Dist-Test
 
 
 ### Components

docs/Continuous-Tests.md

@@ -0,0 +1,119 @@
+# Continuous Tests
+
+ 1. [Description](#description)
+ 2. [Prerequisites](#prerequisites)
+ 3. [Run tests](#run-tests)
+ 4. [Analyze logs](#analyze-logs)
+
+
+## Description
+
+ Continuous Tests were developed to perform long lasting tests in different configurations and topologies. Unlike Distributed Tests, they are running continuously, until we stop them manually. Such approach is very useful to detect the issues which may appear over the time when we may have blocking I/O, unclosed pools/connections and etc.
+
+ Usually, we are running Continuous Tests manually and for automated runs, please refer to the [Tests automation](Automation.md).
+
+ We have two projects in the repository
+ - [CodexNetDeployer](../CodexNetDeployer) - Prepare environment to run the tests
+ - [ContinuousTests](../ContinuousTests) - Continuous Tests
+
+ And they are used to prepare environment and run Continuous Tests.
+
+
+## Prerequisites
+
+ 1. Kubernetes cluster, to run the tests
+ 2. kubeconfig file, to access the cluster
+ 3. [kubectl](https://kubernetes.io/docs/tasks/tools/) installed, to create resources in the cluster
+ 4. Optional - [OpenLens](https://github.com/MuhammedKalkan/OpenLens) installed, to browse cluster resources
+
+
+## Run tests
+ 1. Create a Pod in the cluster, in the `default` namespace and consider to use your own value for `metadata.name`
+    <details>
+    <summary>tests-runner.yaml</summary>
+
+    ```yaml
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: tests-runner
+      namespace: default
+      labels:
+        name: manual-run
+    spec:
+      containers:
+      - name: runner
+        image: mcr.microsoft.com/dotnet/sdk:7.0
+        env:
+        - name: KUBECONFIG
+          value: /opt/kubeconfig.yaml
+      #   volumeMounts:
+      #   - name: kubeconfig
+      #     mountPath: /opt/kubeconfig.yaml
+      #     subPath: kubeconfig.yaml
+      #   - name: logs
+      #    mountPath: /var/log/codex-dist-tests
+        command: ["sleep", "infinity"]
+      # volumes:
+      #   - name: kubeconfig
+      #     secret:
+      #       secretName: codex-dist-tests-app-kubeconfig
+      #   - name: logs
+      #     hostPath:
+      #       path: /var/log/codex-dist-tests
+    ```
+
+    ```shell
+    kubectl apply -f tests-runner.yaml
+    ```
+
+ 2. Copy kubeconfig to the runner Pod using the name you set in the previous step
+    ```shell
+    kubectl cp ~/.kube/codex-dist-tests.yaml tests-runner:/opt/kubeconfig.yaml
+    ```
+
+ 3. Exec into the runner Pod using the name you set in the previous step
+    ```shell
+    # kubectl
+    kubectl exec -it tests-runner -- bash
+
+    # OpenLens
+    OpenLens --> Pods --> dist-tests-runner --> "Press on it" --> Pod Shell
+    ```
+
+ 4. Install required packages
+    ```shell
+    apt update
+    apt install -y tmux
+    ```
+
+ 5. Clone Continuous Tests repository
+    ```shell
+    tmux
+
+    cd /opt
+    git clone https://github.com/codex-storage/cs-codex-dist-tests.git
+    ```
+
+ 6. Run `CodexNetDeployer`
+    ```shell
+    # Usually take ~ 10 minutes
+    cd cs-codex-dist-tests/CodexNetDeployer
+
+    export RUNID=$(date +%Y%m%d-%H%M%S)
+    bash deploy-continuous-testnet.sh
+    ```
+
+ 7. Run `ContinuousTests`
+    ```
+    cd ../ContinuousTests
+    cp ../CodexNetDeployer/codex-deployment.json .
+
+    bash run.sh
+    ```
+
+
+## Analyze logs
+
+ We should check the logs in the `/opt/cs-codex-dist-tests/ContinuousTests/logs` folder