status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/tests

This commit is contained in:
Corbo12 2023-07-18 12:38:19 +02:00
parent a08ef47906 5f89db4012
commit a46fd3b447
66 changed files with 1591 additions and 722 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/docker.yml vendored
View File

@ -78,31 +78,7 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker - Build and export to Docker
uses: docker/build-push-action@v4
with:
context: .
file: ${{ env.DOCKER_FILE }}
platforms: ${{ env.PLATFORM }}
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- name: Docker - Minify image
uses: kitabisa/docker-slim-action@v1
id: slim
env:
DSLIM_HTTP_PROBE: false
with:
target: ${{ steps.meta.outputs.tags }}
overwrite: true
- name: Docker - Show slim report
run: echo "${REPORT}" | jq -r
env:
REPORT: ${{ steps.slim.outputs.report }}
- name: Docker - Push to Docker registry
- name: Docker - Build and Push
uses: docker/build-push-action@v4
with:
context: .
@ -145,4 +121,3 @@ jobs:
inputs: ${{ env.TAGS }}
images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
push: true

33
ArgsUniform/ArgsUniform.cs
View File

@ -4,6 +4,7 @@ namespace ArgsUniform
{
public class ArgsUniform<T>
{
private readonly Action printAppInfo;
private readonly object? defaultsProvider;
private readonly IEnv.IEnv env;
private readonly string[] args;
@ -12,23 +13,24 @@ namespace ArgsUniform
private const int envStart = 48;
private const int descStart = 80;
public ArgsUniform(params string[] args)
: this(new IEnv.Env(), args)
public ArgsUniform(Action printAppInfo, params string[] args)
: this(printAppInfo, new IEnv.Env(), args)
{
}
public ArgsUniform(object defaultsProvider, params string[] args)
: this(defaultsProvider, new IEnv.Env(), args)
public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
: this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
{
}
public ArgsUniform(IEnv.IEnv env, params string[] args)
: this(null!, env, args)
public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
: this(printAppInfo, null!, env, args)
{
}
public ArgsUniform(object defaultsProvider, IEnv.IEnv env, params string[] args)
public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
{
this.printAppInfo = printAppInfo;
this.defaultsProvider = defaultsProvider;
this.env = env;
this.args = args;
@ -36,6 +38,13 @@ namespace ArgsUniform
public T Parse(bool printResult = false)
{
if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
{
printAppInfo();
PrintHelp();
throw new Exception();
}
var result = Activator.CreateInstance<T>();
var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
var missingRequired = new List<PropertyInfo>();
@ -194,6 +203,7 @@ namespace ArgsUniform
{
if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
throw new NotSupportedException();
}
@ -221,6 +231,15 @@ namespace ArgsUniform
return false;
}
private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
{
if (value != null)
{
uniformProperty.SetValue(result, true);
}
return true;
}
private string? GetFromArgs(string key)
{
var argKey = $"--{key}=";

7
ArgsUniform/ExampleUser.cs
View File

@ -21,9 +21,14 @@
// env var: "AAA=BBB"
var args = "--ccc=ddd";
var uniform = new ArgsUniform<Args>(new DefaultsProvider(), args);
var uniform = new ArgsUniform<Args>(PrintHelp, new DefaultsProvider(), args);
var aaa = uniform.Parse();
}
private static void PrintHelp()
{
Console.WriteLine("Help text!");
}
}
}

62
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -2,7 +2,6 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
using Logging;
namespace CodexNetDeployer
{
@ -11,21 +10,17 @@ namespace CodexNetDeployer
private readonly Configuration config;
private readonly WorkflowCreator workflowCreator;
private readonly TestLifecycle lifecycle;
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly GethStartResult gethResult;
private string bootstrapSpr = "";
private int validatorsLeft;
public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators)
public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
{
this.config = config;
this.workflowCreator = workflowCreator;
this.lifecycle = lifecycle;
this.log = log;
this.timeSet = timeSet;
this.gethResult = gethResult;
this.validatorsLeft = numberOfValidators;
validatorsLeft = numberOfValidators;
}
public RunningContainer? Start(int i)
@ -39,24 +34,47 @@ namespace CodexNetDeployer
var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
var container = containers.Containers.First();
var address = lifecycle.Configuration.GetAddress(container);
var codexNode = new CodexNode(log, timeSet, address);
var debugInfo = codexNode.GetDebugInfo();
var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
if (!string.IsNullOrWhiteSpace(debugInfo.spr))
try
{
var pod = container.Pod.PodInfo;
Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine);
var debugInfo = codexAccess.GetDebugInfo();
if (!string.IsNullOrWhiteSpace(debugInfo.spr))
{
Console.Write("Online\t");
if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
validatorsLeft--;
return container;
var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
Console.Write("Tokens minted\t");
var response = marketAccess.MakeStorageAvailable(
totalSpace: config.StorageSell!.Value.MB(),
minPriceForTotalSpace: config.MinPrice.TestTokens(),
maxCollateral: config.MaxCollateral.TestTokens(),
maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
if (!string.IsNullOrEmpty(response))
{
Console.Write("Storage available\tOK" + Environment.NewLine);
if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
validatorsLeft--;
return container;
}
}
}
else
catch (Exception ex)
{
Console.Write("Unknown failure." + Environment.NewLine);
return null;
Console.WriteLine("Exception:" + ex.ToString());
}
Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
lifecycle.DownloadLog(container);
return null;
}
private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
@ -68,6 +86,12 @@ namespace CodexNetDeployer
var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
marketplaceConfig.AccountIndexOverride = i;
codexStart.MarketplaceConfig = marketplaceConfig;
codexStart.MetricsEnabled = config.RecordMetrics;
if (config.BlockTTL != Configuration.SecondsIn1Day)
{
codexStart.BlockTTL = config.BlockTTL;
}
return codexStart;
}

56
CodexNetDeployer/Configuration.cs
View File

@ -1,23 +1,15 @@
using ArgsUniform;
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
namespace CodexNetDeployer
{
public class Configuration
{
[Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
public string CodexImage { get; set; } = string.Empty;
public const int SecondsIn1Day = 24 * 60 * 60;
[Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")]
public string GethImage { get; set; } = string.Empty;
[Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")]
public string ContractsImage { get; set; } = string.Empty;
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")]
public string KubeConfigFile { get; set; } = string.Empty;
[Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
[Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
public string KubeNamespace { get; set; } = string.Empty;
@ -28,22 +20,35 @@ namespace CodexNetDeployer
[Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
public int? NumberOfValidators { get; set; }
[Uniform("storage-quota", "s", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
[Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
public int? StorageQuota { get; set; }
[Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")]
public int? StorageSell { get; set; }
[Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
public CodexLogLevel CodexLogLevel { get; set; }
public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
[Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")]
public int InitialTestTokens { get; set; } = int.MaxValue;
[Uniform("min-price", "mp", "MINPRICE", true, "Minimum price for the storage space for which contracts will be accepted.")]
public int MinPrice { get; set; }
[Uniform("max-collateral", "mc", "MAXCOLLATERAL", true, "Maximum collateral that will be placed for the total storage space.")]
public int MaxCollateral { get; set; }
[Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")]
public int MaxDuration { get; set; }
[Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
public int BlockTTL { get; set; } = SecondsIn1Day;
[Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
public bool RecordMetrics { get; set; } = false;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
public class Defaults
{
public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
}
public List<string> Validate()
{
var errors = new List<string>();
@ -56,6 +61,10 @@ namespace CodexNetDeployer
{
errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
}
if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value)
{
errors.Add("StorageSell cannot be greater than or equal to StorageQuota.");
}
return errors;
}
@ -67,6 +76,7 @@ namespace CodexNetDeployer
{
if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
}
}
@ -74,7 +84,7 @@ namespace CodexNetDeployer
{
if (value == null || value.Value < 1)
{
errors.Add($"{variable} is must be set and must be greater than 0.");
errors.Add($"{variable} must be set and must be greater than 0.");
}
}
@ -82,7 +92,7 @@ namespace CodexNetDeployer
{
if (string.IsNullOrWhiteSpace(value))
{
errors.Add($"{variable} is must be set.");
errors.Add($"{variable} must be set.");
}
}
}

46
CodexNetDeployer/Deployer.cs
View File

@ -27,6 +27,7 @@ namespace CodexNetDeployer
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
var setup = new CodexSetup(999, config.CodexLogLevel);
setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
setup.MetricsEnabled = config.RecordMetrics;
Log("Creating Geth instance and deploying contracts...");
var gethStarter = new GethStarter(lifecycle, workflowCreator);
@ -35,10 +36,15 @@ namespace CodexNetDeployer
Log("Geth started. Codex contracts deployed.");
Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
// It takes a second for the geth node to unlock a single account. Let's assume 3.
// We can't start the codex nodes until their accounts are definitely unlocked. So
// We wait:
Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
Log("Starting Codex nodes...");
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value);
var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexContainers = new List<RunningContainer>();
for (var i = 0; i < config.NumberOfCodexNodes; i++)
{
@ -46,14 +52,18 @@ namespace CodexNetDeployer
if (container != null) codexContainers.Add(container);
}
return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata());
var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
{
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: config.KubeConfigFile,
kubeConfigFile: kubeConfig,
logPath: "null",
logDebug: false,
dataFilesPath: "notUsed",
@ -61,29 +71,45 @@ namespace CodexNetDeployer
runnerLocation: config.RunnerLocation
);
var kubeConfig = new KubernetesWorkflow.Configuration(
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: config.KubeNamespace,
kubeConfigFile: config.KubeConfigFile,
kubeConfigFile: kubeConfig,
operationTimeout: timeset.K8sOperationTimeout(),
retryDelay: timeset.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
return (workflowCreator, lifecycle);
}
private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
{
if (!setup.MetricsEnabled) return null;
Log("Starting metrics service...");
var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
}
private string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
private DeploymentMetadata CreateMetadata()
{
return new DeploymentMetadata(
codexImage: config.CodexImage,
gethImage: config.GethImage,
contractsImage: config.ContractsImage,
kubeNamespace: config.KubeNamespace,
numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
numberOfValidators: config.NumberOfValidators!.Value,
storageQuotaMB: config.StorageQuota!.Value,
codexLogLevel: config.CodexLogLevel);
codexLogLevel: config.CodexLogLevel,
initialTestTokens: config.InitialTestTokens,
minPrice: config.MinPrice,
maxCollateral: config.MaxCollateral,
maxDuration: config.MaxDuration);
}
private void Log(string msg)

27
CodexNetDeployer/Program.cs
View File

@ -1,6 +1,9 @@
using ArgsUniform;
using CodexNetDeployer;
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using Newtonsoft.Json;
using Configuration = CodexNetDeployer.Configuration;
@ -11,13 +14,7 @@ public class Program
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDeployer" + nl);
if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
{
PrintHelp();
return;
}
var uniformArgs = new ArgsUniform<Configuration>(new Configuration.Defaults(), args);
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
var config = uniformArgs.Parse(true);
if (args.Any(a => a == "--external"))
@ -35,6 +32,19 @@ public class Program
return;
}
Console.WriteLine("Using images:" + nl +
$"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
$"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
$"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
$"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
if (!args.Any(a => a == "-y"))
{
Console.WriteLine("Does the above config look good? [y/n]");
if (Console.ReadLine()!.ToLowerInvariant() != "y") return;
Console.WriteLine("I think so too.");
}
var deployer = new Deployer(config);
var deployment = deployer.Deploy();
@ -54,8 +64,5 @@ public class Program
Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
var uniformArgs = new ArgsUniform<Configuration>();
uniformArgs.PrintHelp();
}
}

12
CodexNetDeployer/deploy-continuous-testnet.sh Normal file
View File

@ -0,0 +1,12 @@
dotnet run \
--kube-config=/opt/kubeconfig.yaml \
--kube-namespace=codex-continuous-tests \
--nodes=5 \
--validators=3 \
--storage-quota=2048 \
--storage-sell=1024 \
--min-price=1024 \
--max-collateral=1024 \
--max-duration=3600000 \
--block-ttl=120

16
CodexNetDownloader/CodexNetDownloader.csproj Normal file
View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
</ItemGroup>
</Project>

22
CodexNetDownloader/Configuration.cs Normal file
View File

@ -0,0 +1,22 @@
using ArgsUniform;
using DistTestCore;
using DistTestCore.Codex;
namespace CodexNetDownloader
{
public class Configuration
{
[Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
public string OutputPath { get; set; } = "output";
[Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
public string CodexDeploymentJson { get; set; } = string.Empty;
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
public CodexDeployment CodexDeployment { get; set; } = null!;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
}
}

53
CodexNetDownloader/Program.cs Normal file
View File

@ -0,0 +1,53 @@
using ArgsUniform;
using ContinuousTests;
using DistTestCore;
using DistTestCore.Codex;
using Logging;
using Newtonsoft.Json;
public class Program
{
public static void Main(string[] args)
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDownloader" + nl);
var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
var config = uniformArgs.Parse(true);
if (args.Any(a => a == "--external"))
{
config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
}
config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{
lifecycle.DownloadLog(container);
}
Console.WriteLine("Done!");
}
private static CodexDeployment ParseCodexDeploymentJson(string filename)
{
var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
if (d == null) throw new Exception("Unable to parse " + filename);
return d;
}
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
}
}

20
ContinuousTests/CodexAccessFactory.cs Normal file
View File

@ -0,0 +1,20 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class CodexAccessFactory
{
public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
{
return containers.Select(container =>
{
var address = container.ClusterExternalAddress;
if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
return new CodexAccess(log, container, timeSet, address);
}).ToArray();
}
}
}

19
ContinuousTests/CodexNodeFactory.cs
View File

@ -1,19 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class CodexNodeFactory
{
public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
{
return containers.Select(container =>
{
var address = container.ClusterExternalAddress;
return new CodexNode(log, timeSet, address);
}).ToArray();
}
}
}

27
ContinuousTests/Configuration.cs
View File

@ -1,4 +1,5 @@
using ArgsUniform;
using DistTestCore;
using DistTestCore.Codex;
using Newtonsoft.Json;
@ -18,21 +19,30 @@ namespace ContinuousTests
[Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
public bool KeepPassedTestLogs { get; set; } = false;
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")]
public string KubeConfigFile { get; set; } = string.Empty;
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
public bool StopOnFailure { get; set; } = false;
public CodexDeployment CodexDeployment { get; set; } = null!;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
}
public class ConfigLoader
{
public Configuration Load(string[] args)
{
var uniformArgs = new ArgsUniform<Configuration>(args);
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
var result = uniformArgs.Parse();
var result = uniformArgs.Parse(true);
result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
if (args.Any(a => a == "--external"))
{
result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
}
return result;
}
@ -43,5 +53,14 @@ namespace ContinuousTests
if (d == null) throw new Exception("Unable to parse " + filename);
return d;
}
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
}
}
}

31
ContinuousTests/ContinuousTest.cs
View File

@ -21,23 +21,37 @@ namespace ContinuousTests
private const string UploadFailedMessage = "Unable to store block";
public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration)
public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
{
Nodes = nodes;
Log = log;
FileManager = fileManager;
Configuration = configuration;
CancelToken = cancelToken;
if (nodes != null)
{
NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
}
else
{
NodeRunner = null!;
}
}
public CodexNode[] Nodes { get; private set; } = null!;
public CodexAccess[] Nodes { get; private set; } = null!;
public BaseLog Log { get; private set; } = null!;
public IFileManager FileManager { get; private set; } = null!;
public Configuration Configuration { get; private set; } = null!;
public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
public CancellationToken CancelToken { get; private set; } = new CancellationToken();
public NodeRunner NodeRunner { get; private set; } = null!;
public abstract int RequiredNumberOfNodes { get; }
public abstract TimeSpan RunTestEvery { get; }
public abstract TestFailMode TestFailMode { get; }
public virtual int EthereumAccountIndex { get { return -1; } }
public virtual string CustomK8sNamespace { get { return string.Empty; } }
public string Name
{
@ -47,7 +61,7 @@ namespace ContinuousTests
}
}
public ContentId? UploadFile(CodexNode node, TestFile file)
public ContentId? UploadFile(CodexAccess node, TestFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -57,15 +71,14 @@ namespace ContinuousTests
return node.UploadFile(fileStream);
});
if (response.StartsWith(UploadFailedMessage))
{
return null;
}
if (string.IsNullOrEmpty(response)) return null;
if (response.StartsWith(UploadFailedMessage)) return null;
Log.Log($"Uploaded file. Received contentId: '{response}'.");
return new ContentId(response);
}
public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
var file = FileManager.CreateEmptyTestFile(fileLabel);
@ -74,7 +87,7 @@ namespace ContinuousTests
return file;
}
private void DownloadToFile(CodexNode node, string contentId, TestFile file)
private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
try

48
ContinuousTests/ContinuousTestRunner.cs
View File

@ -1,35 +1,65 @@
using Logging;
using DistTestCore;
using Logging;
namespace ContinuousTests
{
public class ContinuousTestRunner
{
private readonly K8sFactory k8SFactory = new K8sFactory();
private readonly ConfigLoader configLoader = new ConfigLoader();
private readonly TestFactory testFactory = new TestFactory();
private readonly Configuration config;
private readonly StartupChecker startupChecker;
private readonly CancellationToken cancelToken;
public ContinuousTestRunner(string[] args)
public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
{
config = configLoader.Load(args);
startupChecker = new StartupChecker(config);
startupChecker = new StartupChecker(config, cancelToken);
this.cancelToken = cancelToken;
}
public void Run()
{
startupChecker.Check();
var taskFactory = new TaskFactory();
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
overviewLog.Log("Continuous tests starting...");
var allTests = testFactory.CreateTests();
var testStarters = allTests.Select(t => new TestStarter(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
foreach (var t in testStarters)
ClearAllCustomNamespaces(allTests, overviewLog);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
foreach (var testLoop in testLoops)
{
t.Begin();
Thread.Sleep(TimeSpan.FromMinutes(5));
if (cancelToken.IsCancellationRequested) break;
overviewLog.Log("Launching test-loop for " + testLoop.Name);
testLoop.Begin();
Thread.Sleep(TimeSpan.FromSeconds(5));
}
while (true) Thread.Sleep((2 ^ 31) - 1);
overviewLog.Log("Finished launching test-loops.");
cancelToken.WaitHandle.WaitOne();
overviewLog.Log("Cancelling all test-loops...");
taskFactory.WaitAll();
overviewLog.Log("All tasks cancelled.");
}
private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
{
foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
}
private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
{
if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
workflowCreator.CreateWorkflow().DeleteTestResources();
}
}
}

1
ContinuousTests/ContinuousTests.csproj
View File

@ -14,6 +14,7 @@
<ItemGroup>
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
</ItemGroup>

41
ContinuousTests/K8sFactory.cs Normal file
View File

@ -0,0 +1,41 @@
using DistTestCore.Codex;
using DistTestCore;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class K8sFactory
{
public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
{
var kubeConfig = GetKubeConfig(kubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: kubeConfig,
logPath: logPath,
logDebug: false,
dataFilesPath: dataFilePath,
codexLogLevel: CodexLogLevel.Debug,
runnerLocation: runnerLocation
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: customNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
return (workflowCreator, lifecycle);
}
private static string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
}
}

97
ContinuousTests/NodeRunner.cs Normal file
View File

@ -0,0 +1,97 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using DistTestCore;
using KubernetesWorkflow;
using NUnit.Framework;
using Logging;
using Utils;
namespace ContinuousTests
{
public class NodeRunner
{
private readonly K8sFactory k8SFactory = new K8sFactory();
private readonly CodexAccess[] nodes;
private readonly Configuration config;
private readonly ITimeSet timeSet;
private readonly BaseLog log;
private readonly string customNamespace;
private readonly int ethereumAccountIndex;
public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
{
this.nodes = nodes;
this.config = config;
this.timeSet = timeSet;
this.log = log;
this.customNamespace = customNamespace;
this.ethereumAccountIndex = ethereumAccountIndex;
}
public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
{
RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
}
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
{
RunNode(bootstrapNode, operation, 0.TestTokens());
}
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
try
{
var debugInfo = bootstrapNode.GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig();
startupConfig.NameOverride = "TransientNode";
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
startupConfig.Add(config.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
if (mintTestTokens.Amount > 0)
{
var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
}
var container = rc.Containers[0];
var address = lifecycle.Configuration.GetAddress(container);
var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
try
{
operation(codexAccess, marketAccess, lifecycle);
}
catch
{
lifecycle.DownloadLog(container);
throw;
}
}
finally
{
flow.DeleteTestResources();
}
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
{
return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
}
}
}

23
ContinuousTests/Program.cs
View File

@ -6,7 +6,28 @@ public class Program
{
Console.WriteLine("Codex Continous-Test-Runner.");
Console.WriteLine("Running...");
var runner = new ContinuousTestRunner(args);
var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token);
Console.CancelKeyPress += (sender, e) =>
{
Console.WriteLine("Stopping...");
e.Cancel = true;
Cancellation.Cts.Cancel();
};
runner.Run();
Console.WriteLine("Done.");
}
public static class Cancellation
{
static Cancellation()
{
Cts = new CancellationTokenSource();
}
public static CancellationTokenSource Cts { get; }
}
}

126
ContinuousTests/SingleTestRun.cs
View File

@ -4,92 +4,162 @@ using Logging;
using Utils;
using KubernetesWorkflow;
using NUnit.Framework.Internal;
using System.Reflection;
using static Program;
namespace ContinuousTests
{
public class SingleTestRun
{
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
private readonly List<Exception> exceptions = new List<Exception>();
private readonly TaskFactory taskFactory;
private readonly Configuration config;
private readonly BaseLog overviewLog;
private readonly TestHandle handle;
private readonly CodexNode[] nodes;
private readonly CancellationToken cancelToken;
private readonly CodexAccess[] nodes;
private readonly FileManager fileManager;
private readonly FixtureLog fixtureLog;
private readonly string testName;
private readonly string dataFolder;
public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle)
public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
{
this.taskFactory = taskFactory;
this.config = config;
this.overviewLog = overviewLog;
this.handle = handle;
this.cancelToken = cancelToken;
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
}
public void Run()
public void Run(EventWaitHandle runFinishedHandle)
{
Task.Run(() =>
taskFactory.Run(() =>
{
try
{
RunTest();
if (!config.KeepPassedTestLogs) fixtureLog.Delete();
fileManager.DeleteAllTestFiles();
Directory.Delete(dataFolder, true);
runFinishedHandle.Set();
}
catch (Exception ex)
{
fixtureLog.Error("Test run failed with exception: " + ex);
fixtureLog.MarkAsFailed();
overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
Environment.Exit(-1);
}
fileManager.DeleteAllTestFiles();
Directory.Delete(dataFolder, true);
});
}
private void RunTest()
{
try
{
RunTestMoments();
if (!config.KeepPassedTestLogs) fixtureLog.Delete();
}
catch (Exception ex)
{
fixtureLog.Error("Test run failed with exception: " + ex);
fixtureLog.MarkAsFailed();
if (config.StopOnFailure)
{
OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
DownloadClusterLogs();
OverviewLog("Log download finished. Cancelling test runner...");
Cancellation.Cts.Cancel();
}
}
}
private void RunTestMoments()
{
var earliestMoment = handle.GetEarliestMoment();
var t = earliestMoment;
while (true)
{
cancelToken.ThrowIfCancellationRequested();
RunMoment(t);
if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
{
Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
throw exceptions.Single();
ThrowFailTest();
}
var nextMoment = handle.GetNextMoment(t);
if (nextMoment != null)
{
Log($" > Next TestMoment in {nextMoment.Value} seconds...");
t += nextMoment.Value;
Thread.Sleep(nextMoment.Value * 1000);
var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
cancelToken.WaitHandle.WaitOne(delta);
t = nextMoment.Value;
}
else
{
if (exceptions.Any())
{
var ex = exceptions.First();
OverviewLog(" > Test failed: " + ex);
throw ex;
ThrowFailTest();
}
OverviewLog(" > Test passed.");
OverviewLog(" > Test passed. " + FuturesInfo());
return;
}
}
}
private void ThrowFailTest()
{
var ex = UnpackException(exceptions.First());
Log(ex.ToString());
OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
throw ex;
}
private string FuturesInfo()
{
var containers = config.CodexDeployment.CodexContainers;
var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
var f = nodes.Select(n => n.GetDebugFutures().ToString());
var msg = $"(Futures: [{string.Join(", ", f)}])";
return msg;
}
private void DownloadClusterLogs()
{
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{
lifecycle.DownloadLog(container);
}
}
private Exception UnpackException(Exception exception)
{
if (exception is AggregateException a)
{
return UnpackException(a.InnerExceptions.First());
}
if (exception is TargetInvocationException t)
{
return UnpackException(t.InnerException!);
}
return exception;
}
private void RunMoment(int t)
{
using (var context = new TestExecutionContext.IsolatedContext())
@ -100,7 +170,6 @@ namespace ContinuousTests
}
catch (Exception ex)
{
Log($" > TestMoment yielded exception: " + ex);
exceptions.Add(ex);
}
}
@ -111,12 +180,12 @@ namespace ContinuousTests
private void InitializeTest(string name)
{
Log($" > Running TestMoment '{name}'");
handle.Test.Initialize(nodes, fixtureLog, fileManager, config);
handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
}
private void DecommissionTest()
{
handle.Test.Initialize(null!, null!, null!, null!);
handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
}
private void Log(string msg)
@ -127,14 +196,15 @@ namespace ContinuousTests
private void OverviewLog(string msg)
{
Log(msg);
overviewLog.Log(testName + ": " + msg);
var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
overviewLog.Log($"{containerNames} {testName}: {msg}");
}
private CodexNode[] CreateRandomNodes(int number)
private CodexAccess[] CreateRandomNodes(int number)
{
var containers = SelectRandomContainers(number);
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
}
private RunningContainer[] SelectRandomContainers(int number)
@ -151,7 +221,7 @@ namespace ContinuousTests
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
CodexLogLevel.Error, config.RunnerLocation);
}
}
}

94
ContinuousTests/StartupChecker.cs
View File

@ -7,12 +7,14 @@ namespace ContinuousTests
public class StartupChecker
{
private readonly TestFactory testFactory = new TestFactory();
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
private readonly Configuration config;
private readonly CancellationToken cancelToken;
public StartupChecker(Configuration config)
public StartupChecker(Configuration config, CancellationToken cancelToken)
{
this.config = config;
this.cancelToken = cancelToken;
}
public void Check()
@ -35,25 +37,19 @@ namespace ContinuousTests
}
foreach (var test in tests)
{
cancelToken.ThrowIfCancellationRequested();
var handle = new TestHandle(test);
handle.GetEarliestMoment();
handle.GetLastMoment();
}
var errors = new List<string>();
foreach (var test in tests)
{
if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
}
}
if (!Directory.Exists(config.LogPath))
{
Directory.CreateDirectory(config.LogPath);
}
var errors = CheckTests(tests);
if (errors.Any())
{
throw new Exception("Prerun check failed: " + string.Join(", ", errors));
@ -62,13 +58,15 @@ namespace ContinuousTests
private void CheckCodexNodes(BaseLog log, Configuration config)
{
var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
var pass = true;
foreach (var n in nodes)
{
log.Log($"Checking '{n.Address.Host}'...");
cancelToken.ThrowIfCancellationRequested();
if (EnsureOnline(n))
log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'...");
if (EnsureOnline(log, n))
{
log.Log("OK");
}
@ -84,12 +82,14 @@ namespace ContinuousTests
}
}
private bool EnsureOnline(CodexNode n)
private bool EnsureOnline(BaseLog log, CodexAccess n)
{
try
{
var info = n.GetDebugInfo();
if (info == null || string.IsNullOrEmpty(info.id)) return false;
log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
}
catch
{
@ -98,5 +98,69 @@ namespace ContinuousTests
return true;
}
private List<string> CheckTests(ContinuousTest[] tests)
{
var errors = new List<string>();
CheckRequiredNumberOfNodes(tests, errors);
CheckCustomNamespaceClashes(tests, errors);
CheckEthereumIndexClashes(tests, errors);
return errors;
}
private void CheckEthereumIndexClashes(ContinuousTest[] tests, List<string> errors)
{
var offLimits = config.CodexDeployment.CodexContainers.Length;
foreach (var test in tests)
{
if (test.EthereumAccountIndex != -1)
{
if (test.EthereumAccountIndex <= offLimits)
{
errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'.");
}
}
}
DuplicatesCheck(tests, errors,
considerCondition: t => t.EthereumAccountIndex != -1,
getValue: t => t.EthereumAccountIndex,
propertyName: nameof(ContinuousTest.EthereumAccountIndex));
}
private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors)
{
DuplicatesCheck(tests, errors,
considerCondition: t => !string.IsNullOrEmpty(t.CustomK8sNamespace),
getValue: t => t.CustomK8sNamespace,
propertyName: nameof(ContinuousTest.CustomK8sNamespace));
}
private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
{
foreach (var test in tests)
{
if (considerCondition(test))
{
var duplicates = tests.Where(t => t != test && getValue(t) == getValue(test)).ToList();
if (duplicates.Any())
{
duplicates.Add(test);
errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
return;
}
}
}
}
private void CheckRequiredNumberOfNodes(ContinuousTest[] tests, List<string> errors)
{
foreach (var test in tests)
{
if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
}
}
}
}
}

39
ContinuousTests/TaskFactory.cs Normal file
View File

@ -0,0 +1,39 @@
namespace ContinuousTests
{
public class TaskFactory
{
private readonly object taskLock = new();
private readonly List<Task> activeTasks = new List<Task>();
public void Run(Action action)
{
lock (taskLock)
{
activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
}
}
public void WaitAll()
{
var tasks = activeTasks.ToArray();
Task.WaitAll(tasks);
var moreTasks = false;
lock (taskLock)
{
activeTasks.RemoveAll(task => task.IsCompleted);
moreTasks = activeTasks.Any();
}
if (moreTasks) WaitAll();
}
private void CleanupTask(Task completedTask, object? arg)
{
lock (taskLock)
{
activeTasks.Remove(completedTask);
}
}
}
}

67
ContinuousTests/TestLoop.cs Normal file
View File

@ -0,0 +1,67 @@
using Logging;
namespace ContinuousTests
{
public class TestLoop
{
private readonly TaskFactory taskFactory;
private readonly Configuration config;
private readonly BaseLog overviewLog;
private readonly Type testType;
private readonly TimeSpan runsEvery;
private readonly CancellationToken cancelToken;
private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
{
this.taskFactory = taskFactory;
this.config = config;
this.overviewLog = overviewLog;
this.testType = testType;
this.runsEvery = runsEvery;
this.cancelToken = cancelToken;
Name = testType.Name;
}
public string Name { get; }
public void Begin()
{
taskFactory.Run(() =>
{
try
{
while (true)
{
WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle });
cancelToken.ThrowIfCancellationRequested();
StartTest();
cancelToken.WaitHandle.WaitOne(runsEvery);
}
}
catch (OperationCanceledException)
{
overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
}
catch (Exception ex)
{
overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
Environment.Exit(-1);
}
});
}
private void StartTest()
{
var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test);
var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
runFinishedHandle.Reset();
run.Run(runFinishedHandle);
}
}
}

40
ContinuousTests/TestStarter.cs
View File

@ -1,40 +0,0 @@
using Logging;
namespace ContinuousTests
{
public class TestStarter
{
private readonly Configuration config;
private readonly BaseLog overviewLog;
private readonly Type testType;
private readonly TimeSpan runsEvery;
public TestStarter(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
{
this.config = config;
this.overviewLog = overviewLog;
this.testType = testType;
this.runsEvery = runsEvery;
}
public void Begin()
{
Task.Run(() =>
{
while (true)
{
StartTest();
Thread.Sleep(runsEvery);
}
});
}
private void StartTest()
{
var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test);
var run = new SingleTestRun(config, overviewLog, handle);
run.Run();
}
}
}

195
ContinuousTests/Tests/MarketplaceTest.cs
View File

@ -1,130 +1,99 @@
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
//using DistTestCore;
//using DistTestCore.Codex;
//using Newtonsoft.Json;
//using NUnit.Framework;
//using Utils;
namespace ContinuousTests.Tests
{
public class MarketplaceTest : ContinuousTest
{
public override int RequiredNumberOfNodes => 1;
public override TimeSpan RunTestEvery => TimeSpan.FromDays(4);
public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
//namespace ContinuousTests.Tests
//{
// public class MarketplaceTest : ContinuousTest
// {
// public override int RequiredNumberOfNodes => 1;
// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
// public override int EthereumAccountIndex => 200;
// public override string CustomK8sNamespace => "codex-continuous-marketplace";
public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
// private readonly uint numberOfSlots = 3;
// private readonly ByteSize fileSize = 10.MB();
// private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
private const string MarketplaceTestNamespace = "codex-continuous-marketplace";
// private TestFile file = null!;
// private ContentId? cid;
// private string purchaseId = string.Empty;
private readonly ByteSize fileSize = 100.MB();
private readonly TestToken pricePerBytePerSecond = 1.TestTokens();
// [TestMoment(t: Zero)]
// public void NodePostsStorageRequest()
// {
// var contractDuration = TimeSpan.FromMinutes(8);
// decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
// var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000;
private TestFile file = null!;
private ContentId? cid;
private TestToken startingBalance = null!;
private string purchaseId = string.Empty;
// file = FileManager.GenerateTestFile(fileSize);
[TestMoment(t: Zero)]
public void NodePostsStorageRequest()
{
var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds;
// NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
// {
// cid = UploadFile(codexAccess.Node, file);
// Assert.That(cid, Is.Not.Null);
file = FileManager.GenerateTestFile(fileSize);
// purchaseId = marketplaceAccess.RequestStorage(
// contentId: cid!,
// pricePerSlotPerSecond: pricePerSlotPerSecond,
// requiredCollateral: 100.TestTokens(),
// minRequiredNumberOfNodes: numberOfSlots,
// proofProbability: 10,
// duration: contractDuration);
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
startupConfig.Add(codexStartConfig);
startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
// Assert.That(!string.IsNullOrEmpty(purchaseId));
try
{
var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
// WaitForContractToStart(codexAccess, purchaseId);
// });
// }
var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
interaction.MintTestTokens(new[] { account.Account }, expectedTotalCost, tokenAddress);
// [TestMoment(t: MinuteFive + MinuteOne)]
// public void StoredDataIsAvailableAfterThreeDays()
// {
// NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
// {
// var result = DownloadFile(codexAccess.Node, cid!);
var container = rc.Containers[0];
var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork;
var codexAccess = new CodexAccess(lifecycle, container);
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
// file.AssertIsEqual(result);
// });
// }
cid = UploadFile(codexAccess.Node, file);
Assert.That(cid, Is.Not.Null);
// private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId)
// {
// var lastState = "";
// var waitStart = DateTime.UtcNow;
// var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024);
// var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
startingBalance = marketAccess.GetBalance();
// Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}");
// while (lastState != "started")
// {
// CancelToken.ThrowIfCancellationRequested();
purchaseId = marketAccess.RequestStorage(
contentId: cid!,
pricePerBytePerSecond: pricePerBytePerSecond,
requiredCollateral: 100.TestTokens(),
minRequiredNumberOfNodes: 3,
proofProbability: 10,
duration: contractDuration);
// var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
// var statusJson = JsonConvert.SerializeObject(purchaseStatus);
// if (purchaseStatus != null && purchaseStatus.state != lastState)
// {
// lastState = purchaseStatus.state;
// Log.Log("Purchase status: " + statusJson);
// }
Assert.That(!string.IsNullOrEmpty(purchaseId));
}
finally
{
flow.Stop(rc);
}
}
// Thread.Sleep(2000);
[TestMoment(t: DayThree)]
public void StoredDataIsAvailableAfterThreeDays()
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
startupConfig.Add(codexStartConfig);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
// if (lastState == "errored")
// {
// Assert.Fail("Contract start failed: " + statusJson);
// }
try
{
var container = rc.Containers[0];
var codexAccess = new CodexAccess(lifecycle, container);
var result = DownloadContent(codexAccess.Node, cid!);
file.AssertIsEqual(result);
}
finally
{
flow.Stop(rc);
}
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
{
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: Configuration.KubeConfigFile,
logPath: "null",
logDebug: false,
dataFilesPath: "notUsed",
codexLogLevel: CodexLogLevel.Debug,
runnerLocation: TestRunnerLocation.InternalToCluster
);
var kubeConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: MarketplaceTestNamespace,
kubeConfigFile: Configuration.KubeConfigFile,
operationTimeout: TimeSet.K8sOperationTimeout(),
retryDelay: TimeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
return (workflowCreator, lifecycle);
}
}
}
// if (DateTime.UtcNow - waitStart > maxWaitTime)
// {
// Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}");
// }
// }
// Log.Log("Contract started.");
// }
// }
//}

136
ContinuousTests/Tests/PerformanceTests.cs
View File

@ -1,86 +1,86 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
//using DistTestCore;
//using DistTestCore.Codex;
//using NUnit.Framework;
namespace ContinuousTests.Tests
{
public class UploadPerformanceTest : PerformanceTest
{
public override int RequiredNumberOfNodes => 1;
//namespace ContinuousTests.Tests
//{
// public class UploadPerformanceTest : PerformanceTest
// {
// public override int RequiredNumberOfNodes => 1;
[TestMoment(t: Zero)]
public void UploadTest()
{
UploadTest(100, Nodes[0]);
}
}
// [TestMoment(t: Zero)]
// public void UploadTest()
// {
// UploadTest(100, Nodes[0]);
// }
// }
public class DownloadLocalPerformanceTest : PerformanceTest
{
public override int RequiredNumberOfNodes => 1;
// public class DownloadLocalPerformanceTest : PerformanceTest
// {
// public override int RequiredNumberOfNodes => 1;
[TestMoment(t: Zero)]
public void DownloadTest()
{
DownloadTest(100, Nodes[0], Nodes[0]);
}
}
// [TestMoment(t: Zero)]
// public void DownloadTest()
// {
// DownloadTest(100, Nodes[0], Nodes[0]);
// }
// }
public class DownloadRemotePerformanceTest : PerformanceTest
{
public override int RequiredNumberOfNodes => 2;
// public class DownloadRemotePerformanceTest : PerformanceTest
// {
// public override int RequiredNumberOfNodes => 2;
[TestMoment(t: Zero)]
public void DownloadTest()
{
DownloadTest(100, Nodes[0], Nodes[1]);
}
}
// [TestMoment(t: Zero)]
// public void DownloadTest()
// {
// DownloadTest(100, Nodes[0], Nodes[1]);
// }
// }
public abstract class PerformanceTest : ContinuousTest
{
public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
// public abstract class PerformanceTest : ContinuousTest
// {
// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
// public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
public void UploadTest(int megabytes, CodexNode uploadNode)
{
var file = FileManager.GenerateTestFile(megabytes.MB());
// public void UploadTest(int megabytes, CodexAccess uploadNode)
// {
// var file = FileManager.GenerateTestFile(megabytes.MB());
var time = Measure(() =>
{
UploadFile(uploadNode, file);
});
// var time = Measure(() =>
// {
// UploadFile(uploadNode, file);
// });
var timePerMB = time / megabytes;
// var timePerMB = time / megabytes;
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
}
// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
// }
public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode)
{
var file = FileManager.GenerateTestFile(megabytes.MB());
// public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode)
// {
// var file = FileManager.GenerateTestFile(megabytes.MB());
var cid = UploadFile(uploadNode, file);
Assert.That(cid, Is.Not.Null);
// var cid = UploadFile(uploadNode, file);
// Assert.That(cid, Is.Not.Null);
TestFile? result = null;
var time = Measure(() =>
{
result = DownloadContent(downloadNode, cid!);
});
// TestFile? result = null;
// var time = Measure(() =>
// {
// result = DownloadFile(downloadNode, cid!);
// });
file.AssertIsEqual(result);
// file.AssertIsEqual(result);
var timePerMB = time / megabytes;
// var timePerMB = time / megabytes;
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
}
// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
// }
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
}
}
// private static TimeSpan Measure(Action action)
// {
// var start = DateTime.UtcNow;
// action();
// return DateTime.UtcNow - start;
// }
// }
//}

60
ContinuousTests/Tests/ThresholdChecks.cs Normal file
View File

@ -0,0 +1,60 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace ContinuousTests.Tests
{
public class ThresholdChecks : ContinuousTest
{
public override int RequiredNumberOfNodes => 1;
public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private static readonly List<string> previousBreaches = new List<string>();
[TestMoment(t: 0)]
public void CheckAllThresholds()
{
var allNodes = CreateAccessToAllNodes();
foreach (var n in allNodes) CheckThresholds(n);
}
private void CheckThresholds(CodexAccess n)
{
var breaches = n.GetDebugThresholdBreaches();
if (breaches.breaches.Any())
{
var newBreaches = new List<string>();
foreach (var b in breaches.breaches)
{
if (!previousBreaches.Contains(b))
{
newBreaches.Add(b);
previousBreaches.Add(b);
}
}
if (newBreaches.Any())
{
Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
Program.Cancellation.Cts.Cancel();
}
}
}
private string FormatBreach(CodexAccess n, string breach)
{
return $"{n.Container.Name} = '{breach}'";
}
private CodexAccess[] CreateAccessToAllNodes()
{
// Normally, a continuous test accesses only a subset of the nodes in the deployment.
// This time, we want to check all of them.
var factory = new CodexAccessFactory();
var allContainers = Configuration.CodexDeployment.CodexContainers;
return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
}
}
}

54
ContinuousTests/Tests/TransientNodeTest.cs Normal file
View File

@ -0,0 +1,54 @@
//using DistTestCore;
//using DistTestCore.Codex;
//using NUnit.Framework;
//namespace ContinuousTests.Tests
//{
// public class TransientNodeTest : ContinuousTest
// {
// public override int RequiredNumberOfNodes => 3;
// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
// public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant();
// public override int EthereumAccountIndex => 201;
// private TestFile file = null!;
// private ContentId cid = null!;
// private CodexAccess UploadBootstapNode { get { return Nodes[0]; } }
// private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } }
// private CodexAccess IntermediateNode { get { return Nodes[2]; } }
// [TestMoment(t: 0)]
// public void UploadWithTransientNode()
// {
// file = FileManager.GenerateTestFile(10.MB());
// NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
// {
// cid = UploadFile(codexAccess, file)!;
// Assert.That(cid, Is.Not.Null);
// var dlt = Task.Run(() =>
// {
// Thread.Sleep(10000);
// lifecycle.DownloadLog(codexAccess.Container);
// });
// var resultFile = DownloadFile(IntermediateNode, cid);
// dlt.Wait();
// file.AssertIsEqual(resultFile);
// });
// }
// [TestMoment(t: 30)]
// public void DownloadWithTransientNode()
// {
// NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
// {
// var resultFile = DownloadFile(codexAccess, cid);
// file.AssertIsEqual(resultFile);
// });
// }
// }
//}

6
ContinuousTests/Tests/TwoClientTest.cs
View File

@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
public class TwoClientTest : ContinuousTest
{
public override int RequiredNumberOfNodes => 2;
public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private ContentId? cid;
@ -21,10 +21,10 @@ namespace ContinuousTests.Tests
Assert.That(cid, Is.Not.Null);
}
[TestMoment(t: MinuteFive)]
[TestMoment(t: 10)]
public void DownloadTestFile()
{
var dl = DownloadContent(Nodes[1], cid!);
var dl = DownloadFile(Nodes[1], cid!);
file.AssertIsEqual(dl);
}

4
ContinuousTests/run-continuous-tests.sh Normal file
View File

@ -0,0 +1,4 @@
dotnet run \
--kube-config=/opt/kubeconfig.yaml \
--codex-deployment=codex-deployment.json \
--stop=1

5
DistTestCore/AutoBootstrapDistTest.cs
View File

@ -6,7 +6,7 @@ namespace DistTestCore
{
public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
{
throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " +
"If you want to control the bootstrap node from your test, please use DistTest instead.");
}
@ -21,7 +21,8 @@ namespace DistTestCore
[SetUp]
public void SetUpBootstrapNode()
{
BootstrapNode = BringOnline(CreateCodexSetup(1))[0];
var setup = CreateCodexSetup(1).WithName("BOOTSTRAP");
BootstrapNode = BringOnline(setup)[0];
}
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;

94
DistTestCore/Codex/CodexAccess.cs
View File

@ -1,40 +1,92 @@
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore.Codex
{
public class CodexAccess
{
private readonly TestLifecycle lifecycle;
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
{
this.lifecycle = lifecycle;
Container = runningContainer;
var address = lifecycle.Configuration.GetAddress(Container);
Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address);
this.log = log;
Container = container;
this.timeSet = timeSet;
Address = address;
}
public RunningContainer Container { get; }
public CodexNode Node { get; }
public Address Address { get; }
public void EnsureOnline()
public CodexDebugResponse GetDebugInfo()
{
try
{
var debugInfo = Node.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
}
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
catch (Exception e)
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
var http = Http();
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public int GetDebugFutures()
{
// Some Codex images support debug/futures to count the number of open futures.
return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
}
public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
{
return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
}
public string UploadFile(FileStream fileStream)
{
return Http().HttpPostStream("upload", fileStream);
}
public Stream DownloadFile(string contentId)
{
return Http().HttpGetStream("download/" + contentId);
}
public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
{
return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
}
public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
{
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
{
return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http()
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
}
}
}

92
DistTestCore/Codex/CodexNode.cs → DistTestCore/Codex/CodexApiTypes.cs
View File

@ -1,81 +1,9 @@
using Logging;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore.Codex
{
public class CodexNode
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexNode(BaseLog log, ITimeSet timeSet, Address address)
{
this.log = log;
this.timeSet = timeSet;
Address = address;
}
public Address Address { get; }
public CodexDebugResponse GetDebugInfo()
{
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public string UploadFile(FileStream fileStream)
{
return Http().HttpPostStream("upload", fileStream);
}
public Stream DownloadFile(string contentId)
{
return Http().HttpGetStream("download/" + contentId);
}
public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
{
return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
}
public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
{
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http(TimeSpan? timeoutOverride = null)
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
}
}
public class CodexDebugResponse
{
public string id { get; set; } = string.Empty;
@ -88,6 +16,11 @@ namespace DistTestCore.Codex
public CodexDebugTableResponse table { get; set; } = new();
}
public class CodexDebugFutures
{
public int futures { get; set; }
}
public class CodexDebugTableResponse
{
public CodexDebugTableNodeResponse localNode { get; set; } = new();
@ -143,6 +76,11 @@ namespace DistTestCore.Codex
public string address { get; set; } = string.Empty;
}
public class CodexDebugThresholdBreaches
{
public string[] breaches { get; set; } = Array.Empty<string>();
}
public class CodexSalesAvailabilityRequest
{
public string size { get; set; } = string.Empty;
@ -170,4 +108,10 @@ namespace DistTestCore.Codex
public uint? nodes { get; set; }
public uint? tolerance { get; set; }
}
public class CodexStoragePurchase
{
public string state { get; set; } = string.Empty;
public string error { get; set; } = string.Empty;
}
}

55
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -6,10 +6,10 @@ namespace DistTestCore.Codex
public class CodexContainerRecipe : ContainerRecipeFactory
{
#if Arm64
public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
#else
public const string DockerImage = "thatbenbierens/nim-codex:dhting";
//public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
//public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah";
public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
#endif
public const string MetricsPortTag = "metrics_port";
public const string DiscoveryPortTag = "discovery-port";
@ -18,32 +18,51 @@ namespace DistTestCore.Codex
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
protected override string Image => DockerImage;
public static string DockerImageOverride = string.Empty;
protected override string Image
{
get
{
if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
return DockerImage;
}
}
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<CodexStartupConfig>();
AddExposedPortAndVar("API_PORT");
AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag);
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
AddExposedPortAndVar("CODEX_API_PORT");
AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
// This makes the node announce itself to its local (pod) IP address.
AddEnvVar("NAT_IP_AUTO", "true");
var listenPort = AddInternalPort();
AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
if (!string.IsNullOrEmpty(config.BootstrapSpr))
{
AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
}
if (config.StorageQuota != null)
{
AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
}
if (config.BlockTTL != null)
{
AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
}
if (config.MetricsEnabled)
{
AddEnvVar("METRICS_ADDR", "0.0.0.0");
AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
AddEnvVar("CODEX_METRICS", "true");
AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
}
if (config.MarketplaceConfig != null)
@ -56,14 +75,14 @@ namespace DistTestCore.Codex
var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
AddEnvVar("PERSISTENCE", "1");
AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
AddEnvVar("CODEX_ETH_ACCOUNT", companionNodeAccount.Account);
AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
AddEnvVar("CODEX_PERSISTENCE", "true");
if (config.MarketplaceConfig.IsValidator)
{
AddEnvVar("VALIDATOR", "1");
AddEnvVar("CODEX_VALIDATOR", "true");
}
}
}

22
DistTestCore/Codex/CodexDeployment.cs
View File

@ -5,41 +5,45 @@ namespace DistTestCore.Codex
{
public class CodexDeployment
{
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata)
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
{
GethStartResult = gethStartResult;
CodexContainers = codexContainers;
PrometheusContainer = prometheusContainer;
Metadata = metadata;
}
public GethStartResult GethStartResult { get; }
public RunningContainer[] CodexContainers { get; }
public RunningContainer? PrometheusContainer { get; }
public DeploymentMetadata Metadata { get; }
}
public class DeploymentMetadata
{
public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel)
public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
{
DeployDateTimeUtc = DateTime.UtcNow;
CodexImage = codexImage;
GethImage = gethImage;
ContractsImage = contractsImage;
KubeNamespace = kubeNamespace;
NumberOfCodexNodes = numberOfCodexNodes;
NumberOfValidators = numberOfValidators;
StorageQuotaMB = storageQuotaMB;
CodexLogLevel = codexLogLevel;
InitialTestTokens = initialTestTokens;
MinPrice = minPrice;
MaxCollateral = maxCollateral;
MaxDuration = maxDuration;
}
public string CodexImage { get; }
public DateTime DeployDateTimeUtc { get; }
public string GethImage { get; }
public string ContractsImage { get; }
public string KubeNamespace { get; }
public int NumberOfCodexNodes { get; }
public int NumberOfValidators { get; }
public int StorageQuotaMB { get; }
public CodexLogLevel CodexLogLevel { get; }
public int InitialTestTokens { get; }
public int MinPrice { get; }
public int MaxCollateral { get; }
public int MaxDuration { get; }
}
}

1
DistTestCore/Codex/CodexStartupConfig.cs
View File

@ -17,5 +17,6 @@ namespace DistTestCore.Codex
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
public string? BootstrapSpr { get; set; }
public int? BlockTTL { get; set; }
}
}

11
DistTestCore/CodexNodeGroup.cs
View File

@ -64,12 +64,19 @@ namespace DistTestCore
public void EnsureOnline()
{
foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
foreach (var node in Nodes)
{
var debugInfo = node.CodexAccess.GetDebugInfo();
var nodePeerId = debugInfo.id;
var nodeName = node.CodexAccess.Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(lifecycle, c);
var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
return factory.CreateOnlineCodexNode(access, this);
}
}

13
DistTestCore/CodexStarter.cs
View File

@ -1,5 +1,6 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using KubernetesWorkflow;
using Logging;
@ -23,13 +24,13 @@ namespace DistTestCore
var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
var metricAccessFactory = CollectMetrics(codexSetup, containers);
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
var podInfo = group.Containers.RunningPod.PodInfo;
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
LogSeparator();
return group;
}
@ -57,6 +58,14 @@ namespace DistTestCore
workflow.DownloadContainerLog(container, logHandler);
}
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
{
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
}
private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
{
var startupConfig = new StartupConfig();

2
DistTestCore/Configuration.cs
View File

@ -17,7 +17,7 @@ namespace DistTestCore
{
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true";
logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));

2
DistTestCore/DistTest.cs
View File

@ -250,7 +250,7 @@ namespace DistTestCore
{
OnEachCodexNode(lifecycle, node =>
{
lifecycle.DownloadLog(node);
lifecycle.DownloadLog(node.CodexAccess.Container);
});
}

1
DistTestCore/FileManager.cs
View File

@ -1,6 +1,5 @@
using Logging;
using NUnit.Framework;
using System.Runtime.InteropServices;
using Utils;
namespace DistTestCore

26
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -52,12 +52,12 @@ namespace DistTestCore.Helpers
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10);
var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (pairs.Any() && timeout > DateTime.UtcNow)
{
action();
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5));
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
}
}
@ -140,6 +140,12 @@ namespace DistTestCore.Helpers
}
}
public override string ToString()
{
if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
return Response.id;
}
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
{
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
@ -161,7 +167,6 @@ namespace DistTestCore.Helpers
public class Pair
{
private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
@ -188,10 +193,15 @@ namespace DistTestCore.Helpers
return GetResultMessage() + GetTimePostfix();
}
public override string ToString()
{
return $"[{GetMessage()}]";
}
private string GetResultMessage()
{
var aName = A.Response.id;
var bName = B.Response.id;
var aName = A.ToString();
var bName = B.ToString();
if (Success)
{
@ -203,8 +213,8 @@ namespace DistTestCore.Helpers
private string GetTimePostfix()
{
var aName = A.Response.id;
var bName = B.Response.id;
var aName = A.ToString();
var bName = B.ToString();
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
@ -224,7 +234,7 @@ namespace DistTestCore.Helpers
try
{
var response = a.Node.GetDebugPeer(peerId, timeout);
var response = a.Node.GetDebugPeer(peerId);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;

28
DistTestCore/Http.cs
View File

@ -12,15 +12,15 @@ namespace DistTestCore
private readonly ITimeSet timeSet;
private readonly Address address;
private readonly string baseUrl;
private readonly TimeSpan? timeoutOverride;
private readonly string? logAlias;
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null)
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
{
this.log = log;
this.timeSet = timeSet;
this.address = address;
this.baseUrl = baseUrl;
this.timeoutOverride = timeoutOverride;
this.logAlias = logAlias;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
}
@ -113,27 +113,25 @@ namespace DistTestCore
private void Log(string url, string message)
{
log.Debug($"({url}) = '{message}'", 3);
if (logAlias != null)
{
log.Debug($"({logAlias})({url}) = '{message}'", 3);
}
else
{
log.Debug($"({url}) = '{message}'", 3);
}
}
private T Retry<T>(Func<T> operation, string description)
{
return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
}
private HttpClient GetClient()
{
if (timeoutOverride.HasValue)
{
return GetClient(timeoutOverride.Value);
}
return GetClient(timeSet.HttpCallTimeout());
}
private HttpClient GetClient(TimeSpan timeout)
{
var client = new HttpClient();
client.Timeout = timeout;
client.Timeout = timeSet.HttpCallTimeout();
return client;
}
}

10
DistTestCore/Logs/CodexNodeLog.cs → DistTestCore/Logs/DownloadedLog.cs
View File

@ -3,17 +3,17 @@ using NUnit.Framework;
namespace DistTestCore.Logs
{
public interface ICodexNodeLog
public interface IDownloadedLog
{
void AssertLogContains(string expectedString);
}
public class CodexNodeLog : ICodexNodeLog
public class DownloadedLog : IDownloadedLog
{
private readonly LogFile logFile;
private readonly OnlineCodexNode owner;
private readonly string owner;
public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
public DownloadedLog(LogFile logFile, string owner)
{
this.logFile = logFile;
this.owner = owner;
@ -31,7 +31,7 @@ namespace DistTestCore.Logs
line = streamReader.ReadLine();
}
Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
}
}
}

10
DistTestCore/Logs/LogDownloadHandler.cs
View File

@ -5,21 +5,21 @@ namespace DistTestCore.Logs
{
public class LogDownloadHandler : LogHandler, ILogHandler
{
private readonly OnlineCodexNode node;
private readonly RunningContainer container;
private readonly LogFile log;
public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
public LogDownloadHandler(RunningContainer container, string description, LogFile log)
{
this.node = node;
this.container = container;
this.log = log;
log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description);
}
public CodexNodeLog CreateCodexNodeLog()
public DownloadedLog DownloadLog()
{
return new CodexNodeLog(log, node);
return new DownloadedLog(log, container.Name);
}
protected override void ProcessLine(string line)

43
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
public interface IMarketplaceAccess
{
string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
void AssertThatBalance(IResolveConstraint constraint, string message = "");
TestToken GetBalance();
}
@ -30,27 +30,27 @@ namespace DistTestCore.Marketplace
this.codexAccess = codexAccess;
}
public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
{
var request = new CodexSalesRequestStorageRequest
{
duration = ToHexBigInt(duration.TotalSeconds),
proofProbability = ToHexBigInt(proofProbability),
reward = ToHexBigInt(pricePerBytePerSecond),
collateral = ToHexBigInt(requiredCollateral),
duration = ToDecInt(duration.TotalSeconds),
proofProbability = ToDecInt(proofProbability),
reward = ToDecInt(pricePerSlotPerSecond),
collateral = ToDecInt(requiredCollateral),
expiry = null,
nodes = minRequiredNumberOfNodes,
tolerance = null,
};
Log($"Requesting storage for: {contentId.Id}... (" +
$"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
$"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
$"requiredCollateral: {requiredCollateral}, " +
$"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
$"proofProbability: {proofProbability}, " +
$"duration: {Time.FormatDuration(duration)})");
var response = codexAccess.Node.RequestStorage(request, contentId.Id);
var response = codexAccess.RequestStorage(request, contentId.Id);
if (response == "Purchasing not available")
{
@ -62,38 +62,39 @@ namespace DistTestCore.Marketplace
return response;
}
public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration)
public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
{
var request = new CodexSalesAvailabilityRequest
{
size = ToHexBigInt(size.SizeInBytes),
duration = ToHexBigInt(maxDuration.TotalSeconds),
maxCollateral = ToHexBigInt(maxCollateral),
minPrice = ToHexBigInt(minPricePerBytePerSecond)
size = ToDecInt(totalSpace.SizeInBytes),
duration = ToDecInt(maxDuration.TotalSeconds),
maxCollateral = ToDecInt(maxCollateral),
minPrice = ToDecInt(minPriceForTotalSpace)
};
Log($"Making storage available... (" +
$"size: {size}, " +
$"minPricePerBytePerSecond: {minPricePerBytePerSecond}, " +
$"size: {totalSpace}, " +
$"minPriceForTotalSpace: {minPriceForTotalSpace}, " +
$"maxCollateral: {maxCollateral}, " +
$"maxDuration: {Time.FormatDuration(maxDuration)})");
var response = codexAccess.Node.SalesAvailability(request);
var response = codexAccess.SalesAvailability(request);
Log($"Storage successfully made available. Id: {response.id}");
return response.id;
}
private string ToHexBigInt(double d)
private string ToDecInt(double d)
{
return "0x" + string.Format("{0:X}", Convert.ToInt64(d));
var i = new BigInteger(d);
return i.ToString("D");
}
public string ToHexBigInt(TestToken t)
public string ToDecInt(TestToken t)
{
var bigInt = new BigInteger(t.Amount);
return "0x" + bigInt.ToString("X");
var i = new BigInteger(t.Amount);
return i.ToString("D");
}
public void AssertThatBalance(IResolveConstraint constraint, string message = "")

4
DistTestCore/Metrics/MetricsAccess.cs
View File

@ -14,12 +14,12 @@ namespace DistTestCore.Metrics
public class MetricsAccess : IMetricsAccess
{
private readonly TestLog log;
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly MetricsQuery query;
private readonly RunningContainer node;
public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
{
this.log = log;
this.timeSet = timeSet;

4
DistTestCore/Metrics/MetricsDownloader.cs
View File

@ -5,9 +5,9 @@ namespace DistTestCore.Metrics
{
public class MetricsDownloader
{
private readonly TestLog log;
private readonly BaseLog log;
public MetricsDownloader(TestLog log)
public MetricsDownloader(BaseLog log)
{
this.log = log;
}

29
DistTestCore/OnlineCodexNode.cs
View File

@ -12,11 +12,10 @@ namespace DistTestCore
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
ICodexNodeLog DownloadLog();
IDownloadedLog DownloadLog();
IMetricsAccess Metrics { get; }
IMarketplaceAccess Marketplace { get; }
ICodexSetup BringOffline();
@ -49,7 +48,7 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var debugInfo = CodexAccess.Node.GetDebugInfo();
var debugInfo = CodexAccess.GetDebugInfo();
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
return debugInfo;
@ -57,12 +56,7 @@ namespace DistTestCore
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return CodexAccess.Node.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.Node.GetDebugPeer(peerId, timeout);
return CodexAccess.GetDebugPeer(peerId);
}
public ContentId UploadFile(TestFile file)
@ -72,13 +66,12 @@ namespace DistTestCore
var logMessage = $"Uploading file {file.Describe()}...";
var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
{
return CodexAccess.Node.UploadFile(fileStream);
return CodexAccess.UploadFile(fileStream);
});
if (response.StartsWith(UploadFailedMessage))
{
Assert.Fail("Node failed to store block.");
}
if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
var logReplacement = $"(CID:{file.Describe()})";
Log($"ContentId '{response}' is {logReplacement}");
lifecycle.Log.AddStringReplace(response, logReplacement);
@ -101,15 +94,15 @@ namespace DistTestCore
Log($"Connecting to peer {peer.GetName()}...");
var peerInfo = node.GetDebugInfo();
var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
Log($"Successfully connected to peer {peer.GetName()}.");
}
public ICodexNodeLog DownloadLog()
public IDownloadedLog DownloadLog()
{
return lifecycle.DownloadLog(this);
return lifecycle.DownloadLog(CodexAccess.Container);
}
public ICodexSetup BringOffline()
@ -141,7 +134,7 @@ namespace DistTestCore
using var fileStream = File.OpenWrite(file.Filename);
try
{
using var downloadStream = CodexAccess.Node.DownloadFile(contentId);
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
catch

6
DistTestCore/PrometheusStarter.cs
View File

@ -12,10 +12,8 @@ namespace DistTestCore
{
}
public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
public RunningContainers CollectMetricsFor(RunningContainers containers)
{
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
LogStart($"Starting metrics server for {containers.Describe()}");
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
@ -26,7 +24,7 @@ namespace DistTestCore
LogEnd("Metrics server started.");
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
return runningContainers;
}
private string GeneratePrometheusConfig(RunningContainer[] nodes)

16
DistTestCore/TestLifecycle.cs
View File

@ -9,12 +9,12 @@ namespace DistTestCore
{
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
: this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
{
}
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
{
Log = log;
Configuration = configuration;
@ -27,7 +27,7 @@ namespace DistTestCore
testStart = DateTime.UtcNow;
}
public TestLog Log { get; }
public BaseLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; }
@ -41,16 +41,16 @@ namespace DistTestCore
FileManager.DeleteAllTestFiles();
}
public ICodexNodeLog DownloadLog(OnlineCodexNode node)
public IDownloadedLog DownloadLog(RunningContainer container)
{
var subFile = Log.CreateSubfile();
var description = node.GetName();
var handler = new LogDownloadHandler(node, description, subFile);
var description = container.Name;
var handler = new LogDownloadHandler(container, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
CodexStarter.DownloadLog(container, handler);
return new CodexNodeLog(subFile, node);
return new DownloadedLog(subFile, description);
}
public string GetTestDuration()

8
DistTestCore/Timing.cs
View File

@ -10,7 +10,7 @@ namespace DistTestCore
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
TimeSpan HttpCallRetryTimeout();
TimeSpan HttpCallRetryTime();
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
@ -24,7 +24,7 @@ namespace DistTestCore
return TimeSpan.FromSeconds(10);
}
public TimeSpan HttpCallRetryTimeout()
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromMinutes(1);
}
@ -41,7 +41,7 @@ namespace DistTestCore
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(5);
return TimeSpan.FromMinutes(1);
}
public TimeSpan WaitForMetricTimeout()
@ -57,7 +57,7 @@ namespace DistTestCore
return TimeSpan.FromHours(2);
}
public TimeSpan HttpCallRetryTimeout()
public TimeSpan HttpCallRetryTime()
{
return TimeSpan.FromHours(5);
}

11
KubernetesWorkflow/K8sController.cs
View File

@ -39,7 +39,7 @@ namespace KubernetesWorkflow
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var podInfo = FetchNewPod();
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
}
public void Stop(RunningPod pod)
@ -436,9 +436,9 @@ namespace KubernetesWorkflow
#region Service management
private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
{
var result = new Dictionary<ContainerRecipe, Port[]>();
var result = new List<ContainerRecipePortMapEntry>();
var ports = CreateServicePorts(containerRecipes);
@ -468,7 +468,7 @@ namespace KubernetesWorkflow
return (serviceSpec.Metadata.Name, result);
}
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
{
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@ -485,7 +485,8 @@ namespace KubernetesWorkflow
// These service ports belongs to this recipe.
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
result.Add(r, ports);
result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
}
}
}

33
KubernetesWorkflow/RunningPod.cs
View File

@ -2,35 +2,50 @@
{
public class RunningPod
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, ContainerRecipePortMapEntry[] portMapEntries)
{
Cluster = cluster;
PodInfo = podInfo;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
PortMapEntries = portMapEntries;
}
public K8sCluster Cluster { get; }
public PodInfo PodInfo { get; }
public ContainerRecipePortMapEntry[] PortMapEntries { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{
if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
return servicePortMap[containerRecipe];
if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
{
return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
}
return Array.Empty<Port>();
}
}
public class ContainerRecipePortMapEntry
{
public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
{
ContainerNumber = containerNumber;
Ports = ports;
}
public int ContainerNumber { get; }
public Port[] Ports { get; }
}
public class PodInfo
{
public PodInfo(string podName, string podIp, string k8sNodeName)
public PodInfo(string name, string ip, string k8sNodeName)
{
Name = podName;
Ip = podIp;
Name = name;
Ip = ip;
K8SNodeName = k8sNodeName;
}

21
Logging/BaseLog.cs
View File

@ -4,6 +4,7 @@ namespace Logging
{
public abstract class BaseLog
{
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly bool debug;
private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
private bool hasFailed;
@ -14,17 +15,21 @@ namespace Logging
this.debug = debug;
}
protected abstract LogFile CreateLogFile();
protected abstract string GetFullName();
protected LogFile LogFile
public LogFile LogFile
{
get
{
if (logFile == null) logFile = CreateLogFile();
if (logFile == null) logFile = new LogFile(GetFullName(), "log");
return logFile;
}
}
public virtual void EndTest()
{
}
public virtual void Log(string message)
{
LogFile.Write(ApplyReplacements(message));
@ -63,6 +68,11 @@ namespace Logging
File.Delete(LogFile.FullFilename);
}
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
}
private string ApplyReplacements(string str)
{
foreach (var replacement in replacements)
@ -71,6 +81,11 @@ namespace Logging
}
return str;
}
private string GetSubfileNumber()
{
return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
}
}
public class BaseLogStringReplacement

4
Logging/FixtureLog.cs
View File

@ -28,9 +28,9 @@ namespace Logging
Directory.Delete(fullName, true);
}
protected override LogFile CreateLogFile()
protected override string GetFullName()
{
return new LogFile(fullName, "log");
return fullName;
}
private string DetermineFolder(LogConfig config)

4
Logging/NullLog.cs
View File

@ -6,9 +6,9 @@
{
}
protected override LogFile CreateLogFile()
protected override string GetFullName()
{
return null!;
return "NULL";
}
public override void Log(string message)

25
Logging/TestLog.cs
View File

@ -1,11 +1,9 @@
using NUnit.Framework;
using Utils;
namespace Logging
{
public class TestLog : BaseLog
{
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly string methodName;
private readonly string fullName;
@ -18,12 +16,7 @@ namespace Logging
Log($"*** Begin: {methodName}");
}
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile($"{fullName}_{GetSubfileNumber()}", ext);
}
public void EndTest()
public override void EndTest()
{
var result = TestContext.CurrentContext.Result;
@ -40,9 +33,9 @@ namespace Logging
}
}
protected override LogFile CreateLogFile()
protected override string GetFullName()
{
return new LogFile(fullName, "log");
return fullName;
}
private string GetMethodName(string name)
@ -50,12 +43,7 @@ namespace Logging
if (!string.IsNullOrEmpty(name)) return name;
var test = TestContext.CurrentContext.Test;
var args = FormatArguments(test);
return $"{test.MethodName}{args}";
}
private string GetSubfileNumber()
{
return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
return ReplaceInvalidCharacters($"{test.MethodName}{args}");
}
private static string FormatArguments(TestContext.TestAdapter test)
@ -63,5 +51,10 @@ namespace Logging
if (test.Arguments == null || !test.Arguments.Any()) return "";
return $"[{string.Join(',', test.Arguments)}]";
}
private static string ReplaceInvalidCharacters(string name)
{
return name.Replace(":", "_");
}
}
}

22
LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs Normal file
View File

@ -0,0 +1,22 @@
using DistTestCore.Helpers;
using DistTestCore;
using NUnit.Framework;
namespace TestsLong.DownloadConnectivityTests
{
[TestFixture]
public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest
{
[Test]
[UseLongTimeouts]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(10, 15, 20)] int numberOfNodes,
[Values(10, 100)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}
}
}

2
Tests/BasicTests/ExampleTests.cs
View File

@ -64,7 +64,7 @@ namespace Tests.BasicTests
var contentId = buyer.UploadFile(testFile);
buyer.Marketplace.RequestStorage(contentId,
pricePerBytePerSecond: 2.TestTokens(),
pricePerSlotPerSecond: 2.TestTokens(),
requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1,
proofProbability: 5,

24
Tests/BasicTests/ThreeClientTest.cs Normal file
View File

@ -0,0 +1,24 @@
using DistTestCore;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class ThreeClientTest : AutoBootstrapDistTest
{
[Test]
public void ThreeClient()
{
var primary = SetupCodexNode();
var secondary = SetupCodexNode();
var testFile = GenerateTestFile(10.MB());
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
}
}
}

4
Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -9,8 +9,8 @@ namespace Tests.DownloadConnectivityTests
[Test]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(3, 10, 20)] int numberOfNodes,
[Values(1, 10, 100)] int sizeMBs)
[Values(1, 3, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();

1
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -1,7 +1,6 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{

8
cs-codex-dist-testing.sln
View File

@ -21,7 +21,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "Continuo
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -69,6 +71,10 @@ Global
{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

183
docker/README.md
View File

@ -1,59 +1,138 @@
# Run tests with Docker in Kubernetes
We may [run tests localy](../LOCALSETUP.MD) using installed Dotnet and inside Kubernetes we may use a [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags).
We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that.
Custom [entrypoint](docker-entrypoint.sh) will do the following
For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes.
We can run tests on remote cluster in the following ways
#### Run pod inside the cluster using generic .Net image
<details>
<summary>steps</summary>
1. Create dist-tests-runner.yaml
```yaml
--
apiVersion: v1
kind: Pod
metadata:
name: dist-tests-runner
namespace: default
spec:
containers:
- name: dotnet
image: mcr.microsoft.com/dotnet/sdk:7.0
command: ["sleep", "infinity"]
```
2. Deploy pod in the cluster
```shell
kubectl apply -f dist-tests-runner.yaml
```
3. Copy kubeconfig to the pod
```shell
kubectl cp kubeconfig.yaml dist-tests-runner:/opt
```
4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens)
```shell
kubectl exec -it dist-tests-runner -- bash
```
5. Clone repository inside the pod
```shell
git clone https://github.com/codex-storage/cs-codex-dist-tests.git
```
6. Update kubeconfig option in config file
```shell
cd cs-codex-dist-tests
vi DistTestCore/Configuration.cs
```
```dotnet
GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml")
```
7. Run tests
```shell
dotnet test Tests
```
8. Check the results and analyze the logs
</details>
#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags)
Before the run we should create some objects inside the cluster
1. Namespace where we will run the image
2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21)
3. Secret with kubeconfig for created SA
4. Configmap with custom app config if required
For more information please see [Manual run inside Kubernetes via Job](../../../issues/7)
Then we need to create a manifest to run the pod
<details>
<summary>runner.yaml</summary>
```yaml
---
apiVersion: v1
kind: Pod
metadata:
name: dist-tests-runner
namespace: cs-codex-dist-tests
labels:
name: cs-codex-dist-tests
spec:
containers:
- name: cs-codex-dist-tests
image: codexstorage/cs-codex-dist-tests:sha-671ee4e
env:
- name: RUNNERLOCATION
value: InternalToCluster
- name: KUBECONFIG
value: /opt/kubeconfig.yaml
- name: CONFIG
value: "/opt/Configuration.cs"
- name: CONFIG_SHOW
value: "true"
volumeMounts:
- name: kubeconfig
mountPath: /opt/kubeconfig.yaml
subPath: kubeconfig.yaml
- name: config
mountPath: /opt/Configuration.cs
subPath: Configuration.cs
- name: logs
mountPath: /var/log/cs-codex-dist-tests
# command:
# - "dotnet"
# - "test"
# - "Tests"
restartPolicy: Never
volumes:
- name: kubeconfig
secret:
secretName: cs-codex-dist-tests-app-kubeconfig
- name: config
configMap:
name: cs-codex-dist-tests
- name: logs
hostPath:
path: /var/log/cs-codex-dist-tests
```
For more information about pod variables please see [job.yaml](job.yaml).
</details>
And then apply it
```shell
kubectl apply -f runner.yaml
```
After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following
1. Clone repository
2. Switch to the specific branch - `master` by default
3. Run all tests - `dotnet test`
**Run with defaults**
```bash
docker run \
--rm \
--name cs-codex-dist-tests \
codexstorage/cs-codex-dist-tests:sha-686757e
```
**Just short tests**
```bash
docker run \
--rm \
--name cs-codex-dist-tests \
codexstorage/cs-codex-dist-tests:sha-686757e \
dotnet test Tests
```
**Custom branch**
```bash
docker run \
--rm \
--name cs-codex-dist-tests \
--env BRANCH=feature/tests \
codexstorage/cs-codex-dist-tests:sha-686757e
```
**Custom local config**
```bash
docker run \
--rm \
--name cs-codex-dist-tests \
--env CONFIG=/opt/Configuration.cs \
--env CONFIG_SHOW=true \
--volume $PWD/DistTestCore/Configuration.cs:/opt/Configuration.cs \
codexstorage/cs-codex-dist-tests:sha-686757e
```
**Local kubeconfig with custom local config**
```bash
docker run \
--rm \
--name cs-codex-dist-tests \
--env CONFIG=/opt/Configuration.cs \
--env CONFIG_SHOW=true \
--env SOURCE=https://github.com/codex-storage/cs-codex-dist-tests.git \
--volume $PWD/DistTestCore/Configuration.cs:/opt/Configuration.cs \
--volume $PWD/kubeconfig.yml:/opt/kubeconfig.yml \
codexstorage/cs-codex-dist-tests:sha-686757e
```