Merge branch 'master' into spike/local-continuous-debug
# Conflicts: # DistTestCore/Codex/CodexContainerRecipe.cs
This commit is contained in:
benbierens
commit
2803b8b484
|
|
@ -0,0 +1,177 @@
|
|||
name: Docker - Reusable
|
||||
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_file:
|
||||
default: docker/Dockerfile
|
||||
description: Dockerfile
|
||||
required: false
|
||||
type: string
|
||||
docker_repo:
|
||||
default: codexstorage/cs-codex-dist-tests
|
||||
description: DockerHub repository
|
||||
required: false
|
||||
type: string
|
||||
tag_latest:
|
||||
default: true
|
||||
description: Set latest tag for Docker images
|
||||
required: false
|
||||
type: boolean
|
||||
tag_sha:
|
||||
default: true
|
||||
description: Set Git short commit as Docker tag
|
||||
required: false
|
||||
type: boolean
|
||||
tag_suffix:
|
||||
default: ''
|
||||
description: Suffix for Docker images tag
|
||||
required: false
|
||||
type: string
|
||||
|
||||
|
||||
env:
|
||||
DOCKER_FILE: ${{ inputs.docker_file }}
|
||||
DOCKER_REPO: ${{ inputs.docker_repo }}
|
||||
TAG_LATEST: ${{ inputs.tag_latest }}
|
||||
TAG_SHA: ${{ inputs.tag_sha }}
|
||||
TAG_SUFFIX: ${{ inputs.tag_suffix }}
|
||||
|
||||
|
||||
jobs:
|
||||
# Build platform specific image
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
target:
|
||||
- os: linux
|
||||
arch: amd64
|
||||
- os: linux
|
||||
arch: arm64
|
||||
include:
|
||||
- target:
|
||||
os: linux
|
||||
arch: amd64
|
||||
builder: ubuntu-22.04
|
||||
- target:
|
||||
os: linux
|
||||
arch: arm64
|
||||
builder: buildjet-4vcpu-ubuntu-2204-arm
|
||||
|
||||
name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
|
||||
runs-on: ${{ matrix.builder }}
|
||||
env:
|
||||
PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Build and Push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKER_FILE }}
|
||||
platforms: ${{ env.PLATFORM }}
|
||||
push: true
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Docker - Export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Docker - Upload digest
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: digests
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
|
||||
# Publish multi-platform image
|
||||
publish:
|
||||
name: Publish multi-platform image
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Docker - Variables
|
||||
run: |
|
||||
# Adjust custom suffix when set and
|
||||
if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
# Disable SHA tags on tagged release
|
||||
if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
|
||||
echo "TAG_SHA=false" >>$GITHUB_ENV
|
||||
fi
|
||||
# Handle latest and latest-custom using raw
|
||||
if [[ ${{ env.TAG_SHA }} == "false" ]]; then
|
||||
echo "TAG_LATEST=false" >>$GITHUB_ENV
|
||||
echo "TAG_RAW=true" >>$GITHUB_ENV
|
||||
if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
|
||||
else
|
||||
echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
else
|
||||
echo "TAG_RAW=false" >>$GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Docker - Download digests
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: digests
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
flavor: |
|
||||
latest=${{ env.TAG_LATEST }}
|
||||
suffix=${{ env.TAG_SUFFIX }},onlatest=true
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,enable=${{ env.TAG_RAW }},value=latest
|
||||
type=sha,enable=${{ env.TAG_SHA }}
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
|
||||
|
||||
- name: Docker - Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}
|
||||
|
|
@ -11,113 +11,12 @@ on:
|
|||
- docker/Dockerfile
|
||||
- docker/docker-entrypoint.sh
|
||||
- .github/workflows/docker.yml
|
||||
- .github/workflows/docker-reusable.yml
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
env:
|
||||
DOCKER_FILE: docker/Dockerfile
|
||||
DOCKER_REPO: codexstorage/cs-codex-dist-tests
|
||||
|
||||
|
||||
jobs:
|
||||
# Build platform specific image
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
target:
|
||||
- os: linux
|
||||
arch: amd64
|
||||
- os: linux
|
||||
arch: arm64
|
||||
include:
|
||||
- target:
|
||||
os: linux
|
||||
arch: amd64
|
||||
builder: ubuntu-22.04
|
||||
- target:
|
||||
os: linux
|
||||
arch: arm64
|
||||
builder: buildjet-4vcpu-ubuntu-2204-arm
|
||||
|
||||
name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
|
||||
runs-on: ${{ matrix.builder }}
|
||||
outputs:
|
||||
tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
|
||||
tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
|
||||
env:
|
||||
PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
|
||||
SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
flavor: |
|
||||
latest=false
|
||||
tags: |
|
||||
type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
|
||||
type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
- name: Docker - Set tags output
|
||||
id: tags
|
||||
run: |
|
||||
if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
|
||||
echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
|
||||
elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
|
||||
echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Build and Push
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKER_FILE }}
|
||||
platforms: ${{ env.PLATFORM }}
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
# Publish single image
|
||||
publish:
|
||||
name: Push single image
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
- name: Docker - Set tags
|
||||
run: |
|
||||
# Transform multi-line tags in to the comma-seperated
|
||||
TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
|
||||
echo "TAGS=${TAGS}" >>$GITHUB_ENV
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Create and push manifest images
|
||||
uses: Noelware/docker-manifest-action@master
|
||||
with:
|
||||
inputs: ${{ env.TAGS }}
|
||||
images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
|
||||
push: true
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
secrets: inherit
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ namespace DistTestCore.Codex
|
|||
{
|
||||
public class CodexContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
private const string DefaultDockerImage = "thatbenbierens/nim-codex:updownload";
|
||||
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
|
||||
|
||||
public const string MetricsPortTag = "metrics_port";
|
||||
public const string DiscoveryPortTag = "discovery-port";
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ namespace DistTestCore
|
|||
private readonly string dataFilesPath;
|
||||
private readonly CodexLogLevel codexLogLevel;
|
||||
private readonly string k8sNamespacePrefix;
|
||||
private RunnerLocation? runnerLocation = null;
|
||||
private static RunnerLocation? runnerLocation = null;
|
||||
|
||||
public Configuration()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ namespace DistTestCore.Marketplace
|
|||
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
|
||||
|
||||
public override string AppName => "codex-contracts";
|
||||
public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc";
|
||||
public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ namespace DistTestCore.Marketplace
|
|||
public const string AccountsFilename = "accounts.csv";
|
||||
|
||||
public override string AppName => "geth";
|
||||
public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d";
|
||||
public override string Image => "codexstorage/dist-tests-geth:latest";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ namespace DistTestCore.Metrics
|
|||
public class PrometheusContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
public override string AppName => "prometheus";
|
||||
public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd";
|
||||
public override string Image => "codexstorage/dist-tests-prometheus:latest";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,12 +37,20 @@ namespace KubernetesWorkflow
|
|||
|
||||
private void Add(string key, string value)
|
||||
{
|
||||
labels.Add(key,
|
||||
value.ToLowerInvariant()
|
||||
.Replace(":","-")
|
||||
labels.Add(key, Format(value));
|
||||
}
|
||||
|
||||
private static string Format(string s)
|
||||
{
|
||||
var result = s.ToLowerInvariant()
|
||||
.Replace(":", "-")
|
||||
.Replace("/", "-")
|
||||
.Replace("\\", "-")
|
||||
);
|
||||
.Replace("[", "-")
|
||||
.Replace("]", "-")
|
||||
.Replace(",", "-");
|
||||
|
||||
return result.Trim('-');
|
||||
}
|
||||
|
||||
internal Dictionary<string, string> GetLabels()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
|
||||
Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
|
||||
|
||||
|
||||
Nim-Codex: https://github.com/codex-storage/nim-codex
|
||||
Dotnet: v6.0
|
||||
Kubernetes: v1.25.4
|
||||
|
|
@ -23,7 +22,7 @@ Test executing can be configured using the following environment variables.
|
|||
|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
|
||||
| KUBECONFIG | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available. | (null) |
|
||||
| LOGPATH | Path (abs or rel) where log files will be saved. | "CodexTestLogs" |
|
||||
| LOGDEBUG | When "true", enables additional test-runner debug log output. | "false" |
|
||||
| LOGDEBUG | When "true", enables additional test-runner debug log output. | "false" |
|
||||
| DATAFILEPATH | Path (abs or rel) where temporary test data files will be saved. | "TestDataFiles" |
|
||||
| LOGLEVEL | Codex log-level. (case-insensitive) | "Trace" |
|
||||
| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
|
||||
|
|
@ -32,10 +31,10 @@ Test executing can be configured using the following environment variables.
|
|||
Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
|
||||
|
||||
## How to contribute tests
|
||||
An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
|
||||
An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.md).
|
||||
|
||||
## Run the tests on your machine
|
||||
Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
|
||||
Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/docs/LOCALSETUP.md). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
|
||||
|
||||
## Missing functionality
|
||||
Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.
|
||||
|
|
|
|||
138
docker/README.md
138
docker/README.md
|
|
@ -1,138 +0,0 @@
|
|||
# Run tests with Docker in Kubernetes
|
||||
|
||||
We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that.
|
||||
|
||||
|
||||
For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes.
|
||||
|
||||
We can run tests on remote cluster in the following ways
|
||||
|
||||
#### Run pod inside the cluster using generic .Net image
|
||||
|
||||
<details>
|
||||
<summary>steps</summary>
|
||||
|
||||
1. Create dist-tests-runner.yaml
|
||||
```yaml
|
||||
--
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: dist-tests-runner
|
||||
namespace: default
|
||||
spec:
|
||||
containers:
|
||||
- name: dotnet
|
||||
image: mcr.microsoft.com/dotnet/sdk:7.0
|
||||
command: ["sleep", "infinity"]
|
||||
```
|
||||
2. Deploy pod in the cluster
|
||||
```shell
|
||||
kubectl apply -f dist-tests-runner.yaml
|
||||
```
|
||||
|
||||
3. Copy kubeconfig to the pod
|
||||
```shell
|
||||
kubectl cp kubeconfig.yaml dist-tests-runner:/opt
|
||||
```
|
||||
|
||||
4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens)
|
||||
```shell
|
||||
kubectl exec -it dist-tests-runner -- bash
|
||||
```
|
||||
|
||||
5. Clone repository inside the pod
|
||||
```shell
|
||||
git clone https://github.com/codex-storage/cs-codex-dist-tests.git
|
||||
```
|
||||
|
||||
6. Update kubeconfig option in config file
|
||||
```shell
|
||||
cd cs-codex-dist-tests
|
||||
vi DistTestCore/Configuration.cs
|
||||
```
|
||||
```dotnet
|
||||
GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml")
|
||||
```
|
||||
|
||||
7. Run tests
|
||||
```shell
|
||||
dotnet test Tests
|
||||
```
|
||||
|
||||
8. Check the results and analyze the logs
|
||||
</details>
|
||||
|
||||
#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags)
|
||||
|
||||
Before the run we should create some objects inside the cluster
|
||||
1. Namespace where we will run the image
|
||||
2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21)
|
||||
3. Secret with kubeconfig for created SA
|
||||
4. Configmap with custom app config if required
|
||||
|
||||
For more information please see [Manual run inside Kubernetes via Job](../../../issues/7)
|
||||
|
||||
Then we need to create a manifest to run the pod
|
||||
<details>
|
||||
<summary>runner.yaml</summary>
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: dist-tests-runner
|
||||
namespace: cs-codex-dist-tests
|
||||
labels:
|
||||
name: cs-codex-dist-tests
|
||||
spec:
|
||||
containers:
|
||||
- name: cs-codex-dist-tests
|
||||
image: codexstorage/cs-codex-dist-tests:sha-671ee4e
|
||||
env:
|
||||
- name: RUNNERLOCATION
|
||||
value: InternalToCluster
|
||||
- name: KUBECONFIG
|
||||
value: /opt/kubeconfig.yaml
|
||||
- name: CONFIG
|
||||
value: "/opt/Configuration.cs"
|
||||
- name: CONFIG_SHOW
|
||||
value: "true"
|
||||
volumeMounts:
|
||||
- name: kubeconfig
|
||||
mountPath: /opt/kubeconfig.yaml
|
||||
subPath: kubeconfig.yaml
|
||||
- name: config
|
||||
mountPath: /opt/Configuration.cs
|
||||
subPath: Configuration.cs
|
||||
- name: logs
|
||||
mountPath: /var/log/cs-codex-dist-tests
|
||||
# command:
|
||||
# - "dotnet"
|
||||
# - "test"
|
||||
# - "Tests"
|
||||
restartPolicy: Never
|
||||
volumes:
|
||||
- name: kubeconfig
|
||||
secret:
|
||||
secretName: cs-codex-dist-tests-app-kubeconfig
|
||||
- name: config
|
||||
configMap:
|
||||
name: cs-codex-dist-tests
|
||||
- name: logs
|
||||
hostPath:
|
||||
path: /var/log/cs-codex-dist-tests
|
||||
```
|
||||
For more information about pod variables please see [job.yaml](job.yaml).
|
||||
</details>
|
||||
|
||||
And then apply it
|
||||
```shell
|
||||
kubectl apply -f runner.yaml
|
||||
```
|
||||
|
||||
After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following
|
||||
1. Clone repository
|
||||
2. Switch to the specific branch - `master` by default
|
||||
3. Run all tests - `dotnet test`
|
||||
|
|
@ -14,7 +14,8 @@ spec:
|
|||
spec:
|
||||
containers:
|
||||
- name: ${NAMEPREFIX}-runner
|
||||
image: codexstorage/cs-codex-dist-tests:sha-300b91e
|
||||
image: codexstorage/cs-codex-dist-tests:latest
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: RUNNERLOCATION
|
||||
value: InternalToCluster
|
||||
|
|
|
|||
|
|
@ -0,0 +1,253 @@
|
|||
# Distributed Tests automation
|
||||
|
||||
1. [Description](#description)
|
||||
2. [Architecture](#architecture)
|
||||
3. [Run on remote cluster](#run-on-remote-cluster)
|
||||
- [Kubernetes cluster pre-configuration](#kubernetes-cluster-pre-configuration)
|
||||
- [Run tests manually](#run-tests-manually)
|
||||
- [Run tests automatically](#run-tests-automatically)
|
||||
- [Tests logs](#tests-logs)
|
||||
|
||||
|
||||
## Description
|
||||
|
||||
The goal of [Distributed System Tests for Nim-Codex](../../) is to test how [Codex](https://codex.storage) works in different topologies in the distributed network and to be able to detect regressions during development.
|
||||
|
||||
We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster.
|
||||
|
||||
Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](/ContinuousTests) which are called to detect issues on continuous Codex runs.
|
||||
|
||||
|
||||
## Architecture
|
||||
|
||||
<img src="Architecture.png" alt="Architecture" width="800"/>
|
||||
|
||||
```
|
||||
Logs --> Kibana
|
||||
/ |
|
||||
GitHub --> CI --> Kubernetes --> Job Prometheus Elaticsearch
|
||||
\ / \ / \ |
|
||||
------------------------ Metrics --> Grafana
|
||||
```
|
||||
|
||||
|
||||
### Components
|
||||
|
||||
| Component | Description |
|
||||
| --------------------------------------- | --------------------------------------- |
|
||||
| [cs-codex-dist-tests](/) | Distributed System Tests |
|
||||
| [Kubernetes](https://kubernetes.io) | Environment where we run Tests |
|
||||
| [Vector](https://vector.dev) | Ship logs to the Elasticsearch |
|
||||
| [Elasticsearch](https://www.elastic.co) | Store and index logs |
|
||||
| [Kibana](https://www.elastic.co) | Discover the logs in Elasticsearch |
|
||||
| [Grafana](https://grafana.com) | Visualize tests run results and metrics |
|
||||
| [Prometheus](https://prometheus.io) | Collect and store metrics |
|
||||
|
||||
> Note: These components are not mentioned on the diagram and provided to understand what do we have under the hood
|
||||
|
||||
|
||||
## Run on remote cluster
|
||||
|
||||
In case of local run we use [Docker Desktop Kubernetes cluster](https://docs.docker.com/desktop/kubernetes/) and during the services checks, app connect directly to the cluster worker nodes and perform ports check. And in case of remote cluster, it would be required to configure services ports exposing using Ingress Controller or run tests directly inside the cluster.
|
||||
|
||||
Now, we have a configuration key `RUNNERLOCATION` which change the logic of the services ports check and when we run tests inside remote cluster we should set it to the `InternalToCluster`.
|
||||
|
||||
As for now, it was decided to run tests inside the remote Kubernetes cluster using CI because
|
||||
- Stable connection from app to the Kubernetes API
|
||||
- Independence from the CI limitations for long runs
|
||||
- Easy, fast, configurable and detachable run
|
||||
|
||||
Because tests are run on remote cluster we need a way to see their execution status and to analyze the logs as well. For that we use Elasticsearch, Kibana and Grafana with logs shipped by Vector. Please see [Tests logs](#tests-logs) for more information.
|
||||
|
||||
Now we can [Run tests manually](#run-tests-manually) and [Run tests automatically](#run-tests-automatically) on [remote Kubernetes cluster which requires to be pre-configured](#kubernetes-cluster-pre-configuration).
|
||||
|
||||
|
||||
### Kubernetes cluster pre-configuration
|
||||
|
||||
Before running the tests on remote Kubernetes cluster we performed [manual pre-configuration](../../../issues/7) and it was require to
|
||||
1. [Create a namespace](../../../issues/7)
|
||||
2. [Create kubeconfig for App](../../../issues/21)
|
||||
3. [Create a secret with created kubeconfig](../../../issues/21)
|
||||
4. [Create kubeconfig for GitHub Actions](../../../issues/19)
|
||||
|
||||
|
||||
### Run tests manually
|
||||
|
||||
**To run tests manually we have the following requirements**
|
||||
1. Get kubeconfig - to access the cluster
|
||||
2. Install [kubectl](https://kubernetes.io/docs/tasks/tools/) - to create resources in the cluster
|
||||
3. Install [OpenLens](https://github.com/MuhammedKalkan/OpenLens) - to browse cluster resources
|
||||
|
||||
**And to run the tests we should perform the following steps**
|
||||
1. Create a Pod in the cluster, in the `default` namespace and consider to use your own value for `metadata.name`
|
||||
<details>
|
||||
<summary>pod.yaml</summary>
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: dist-tests-runner
|
||||
namespace: default
|
||||
labels:
|
||||
app: dist-tests-runner
|
||||
launched: manually
|
||||
spec:
|
||||
containers:
|
||||
- name: dotnet
|
||||
image: mcr.microsoft.com/dotnet/sdk:7.0
|
||||
env:
|
||||
- name: RUNNERLOCATION
|
||||
value: InternalToCluster
|
||||
- name: KUBECONFIG
|
||||
value: /opt/kubeconfig.yaml
|
||||
command: ["sleep", "infinity"]
|
||||
volumeMounts:
|
||||
- name: kubeconfig
|
||||
mountPath: /opt/kubeconfig.yaml
|
||||
subPath: kubeconfig.yaml
|
||||
- name: logs
|
||||
mountPath: /var/log/cs-codex-dist-tests
|
||||
restartPolicy: Never
|
||||
volumes:
|
||||
- name: kubeconfig
|
||||
secret:
|
||||
secretName: cs-codex-dist-tests-app-kubeconfig
|
||||
- name: logs
|
||||
hostPath:
|
||||
path: /var/log/cs-codex-dist-tests
|
||||
```
|
||||
</details>
|
||||
|
||||
```shell
|
||||
kubectl apply -f pod.yaml
|
||||
```
|
||||
|
||||
2. Exec into the Pod using the name you set in the previous step
|
||||
```shell
|
||||
# kubectl
|
||||
kubectl exec -it dist-tests-runner -- bash
|
||||
|
||||
# OpenLens
|
||||
OpenLens --> Pods --> dist-tests-runner --> "Press on it" --> Pod Shell
|
||||
```
|
||||
|
||||
3. Clone dist-tests repository
|
||||
```shell
|
||||
folder="/opt/dist-tests"
|
||||
|
||||
git clone https://github.com/codex-storage/cs-codex-dist-tests.git $folder
|
||||
cd $folder
|
||||
```
|
||||
|
||||
4. Define variables - optional
|
||||
```shell
|
||||
# RUNNERLOCATION # defined at Pod creation
|
||||
# KUBECONFIG # defined at Pod creation
|
||||
export LOGPATH="/var/log/cs-codex-dist-tests" # Logs from that location will be send in Elasticsearch
|
||||
export RUNID=$(date +%Y%m%d-%H%M%S) # Run ID to show in Kibana/Grafana
|
||||
export TESTID=$(git rev-parse --short HEAD) # Test ID to show in Kibana/Grafana
|
||||
```
|
||||
|
||||
5. Run tests
|
||||
```shell
|
||||
# All tests
|
||||
dotnet test
|
||||
|
||||
# Short tests
|
||||
dotnet test Tests
|
||||
|
||||
# Long tests
|
||||
dotnet test LongTests
|
||||
|
||||
# Specific test
|
||||
dotnet test --filter=CodexLogExample
|
||||
```
|
||||
|
||||
6. We can see in OpenLens Pods started by dist-tests app
|
||||
|
||||
7. If we set `LOGPATH` to a location specified above, we should be able to see tests execution status in Kibana/Grafana. For more information, please see [Tests logs](#tests-logs).
|
||||
|
||||
|
||||
### Run tests automatically
|
||||
|
||||
Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge.
|
||||
|
||||
**It works in the following way**
|
||||
1. Github Actions secrets contains [kubeconfig to interact with the Kubernetes cluster](../../../issues/19).
|
||||
2. GitHub Actions [workflow](../../../actions/workflows/dist-tests.yaml) uses kubectl to create Kubernetes Job based on the [job.yaml](/docker/job.yaml) manifest file. It also accept optional inputs at run
|
||||
- `source` - Dist-tests source repository
|
||||
- `branch` - Repository branch
|
||||
- `namespace` - Namespace where Dist-test runner will de created
|
||||
- `nameprefix` - Dist-test runner name prefix
|
||||
|
||||
3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [codexstorage/cs-codex-dist-tests](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags).
|
||||
Image [entrypoint](/docker/docker-entrypoint.sh) is customizable and we can pass the following variables
|
||||
- `SOURCE` - Dist-tests source repository, useful when we work with forks - `default="current repository"`
|
||||
- `BRANCH` - Repository branch, useful when we would like to run tests from the custom branch - `default="master"`
|
||||
- `FOLDER` - Where to clone repository and it is done just to organize the things - `default="/opt/dist-tests"`
|
||||
|
||||
> **Note:** Variables `SOURCE` and `BRANCH` passed by GitHub Actions to the Kubernetes Job and then to the Pod.
|
||||
|
||||
4. Job manifest is setting all required variables which are part of the Pod run
|
||||
```shell
|
||||
RUNNERLOCATION=InternalToCluster
|
||||
KUBECONFIG=/opt/kubeconfig.yaml
|
||||
LOGPATH=/var/log/cs-codex-dist-tests
|
||||
NAMESPACE=default
|
||||
SOURCE=current repository
|
||||
BRANCH=master
|
||||
RUNID=datetime
|
||||
TESTID=short sha
|
||||
```
|
||||
|
||||
5. Dist-tests runner will use [kubeconfig to interact with the Kubernetes and run the tests](../../../issues/21), which is set by `KUBECONFIG` variable.
|
||||
|
||||
6. Runner will execute all tests and will write the logs to the `/var/log/cs-codex-dist-tests` folder.
|
||||
|
||||
7. Kubernetes Job status will be changed from `Running` to the `Completed` or `Failed`.
|
||||
|
||||
8. Vector will ship the logs to the Elasticsearch.
|
||||
|
||||
9. We can check execution status in Kibana and Grafana. For more information, please see [Tests logs](#tests-logs).
|
||||
|
||||
|
||||
### Tests logs
|
||||
|
||||
> **Note:** This part is not finished yet and it is under development
|
||||
|
||||
We use Elasticsearch to store and discover the logs of the tests execution and Codex Pods which are run during the tests.
|
||||
|
||||
We can use [Kibana](#kibana) to discover all the logs and [Grafana](#grafana) to see tests execution status.
|
||||
|
||||
|
||||
#### Endpoints
|
||||
|
||||
| App | URL | Authentication |
|
||||
| ------- | ---------------------------------------------------------------------------- | -------------- |
|
||||
| Kibana | [kibana.dist-tests.codex.storage](https://kibana.dist-tests.codex.storage) | GitHub account |
|
||||
| Grafana | [grafana.dist-tests.codex.storage](https://grafana.dist-tests.codex.storage) | GitHub account |
|
||||
|
||||
|
||||
#### Kibana
|
||||
|
||||
As for now, we have the following indices in Kibana
|
||||
| # | Index Pattern | Description |
|
||||
| - | -------------------------- | ----------------------------------- |
|
||||
| 1 | `dist-tests-status*` | Dist-tests execution status logs |
|
||||
| 2 | `dist-tests-logs*` | Dist-tests logs collected by runner |
|
||||
| 3 | `dist-tests-pods*` | Dist-tests Pods logs |
|
||||
| 4 | `continuous-tests-status*` | Dist-tests execution status logs |
|
||||
| 5 | `continuous-tests-logs*` | Dist-tests logs collected by runner |
|
||||
| 6 | `continuous-tests-pods*` | Dist-tests Pods logs |
|
||||
| 7 | `kubernetes*` | All Kubernetes Pods logs |
|
||||
|
||||
|
||||
#### Grafana
|
||||
|
||||
As for now, we have the following Dashboards in Grafana
|
||||
| # | Dashboard |
|
||||
| - | -------------------------- |
|
||||
| 1 | Distributed Tests - Status |
|
||||
| 2 | Continuous Tests - Status |
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 537 KiB |
Loading…
Reference in New Issue