status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/codex-net-deployer

This commit is contained in:
benbierens 2023-06-27 15:28:35 +02:00
parent 2e09c9135e 63068aae1d
commit ccd5119b06
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
39 changed files with 1117 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

235
ArgsUniform/ArgsUniform.cs Normal file
View File

@ -0,0 +1,235 @@
using System.Reflection;
namespace ArgsUniform
{
public class ArgsUniform<T>
{
private readonly object? defaultsProvider;
private readonly IEnv.IEnv env;
private readonly string[] args;
private const int cliStart = 8;
private const int shortStart = 38;
private const int envStart = 48;
private const int descStart = 80;
public ArgsUniform(params string[] args)
: this(new IEnv.Env(), args)
{
}
public ArgsUniform(object defaultsProvider, params string[] args)
: this(defaultsProvider, new IEnv.Env(), args)
{
}
public ArgsUniform(IEnv.IEnv env, params string[] args)
: this(null!, env, args)
{
}
public ArgsUniform(object defaultsProvider, IEnv.IEnv env, params string[] args)
{
this.defaultsProvider = defaultsProvider;
this.env = env;
this.args = args;
}
public T Parse(bool printResult = false)
{
var result = Activator.CreateInstance<T>();
var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
var missingRequired = new List<PropertyInfo>();
foreach (var uniformProperty in uniformProperties)
{
var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
if (attr != null)
{
if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
{
{
missingRequired.Add(uniformProperty);
}
}
}
}
if (missingRequired.Any())
{
PrintResults(result, uniformProperties);
Print("");
foreach (var missing in missingRequired)
{
var attr = missing.GetCustomAttribute<UniformAttribute>()!;
var exampleArg = $"--{attr.Arg}=...";
var exampleEnvVar = $"{attr.EnvVar}=...";
Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
}
PrintHelp();
throw new ArgumentException("Unable to assemble all required arguments");
}
if (printResult)
{
PrintResults(result, uniformProperties);
}
return result;
}
private void PrintResults(T result, PropertyInfo[] uniformProperties)
{
Print("");
foreach (var p in uniformProperties)
{
Print($"\t{p.Name} = {p.GetValue(result)}");
}
Print("");
}
public void PrintHelp()
{
Print("");
PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
foreach (var attr in attrs)
{
var a = attr!;
var optional = !a.Required ? " *" : "";
PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
}
Print("");
}
private void Print(string msg)
{
Console.WriteLine(msg);
}
private void PrintAligned(string cli, string s, string env, string desc)
{
Console.CursorLeft = cliStart;
Console.Write(cli);
Console.CursorLeft = shortStart;
Console.Write(s);
Console.CursorLeft = envStart;
Console.Write(env);
Console.CursorLeft = descStart;
Console.Write(desc + Environment.NewLine);
}
private object GetDefaultValue(Type t)
{
if (t.IsValueType) return Activator.CreateInstance(t)!;
return null!;
}
private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
{
if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
return false;
}
private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
{
var currentValue = uniformProperty.GetValue(result);
var isEmptryString = (currentValue as string) == string.Empty;
if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
if (defaultsProvider == null) return false;
var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
if (defaultProperty == null) return false;
var value = defaultProperty.GetValue(defaultsProvider);
if (value != null)
{
return Assign(result, uniformProperty, value);
}
return false;
}
private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
{
var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
if (!string.IsNullOrEmpty(e))
{
return Assign(result, uniformProperty, e);
}
return false;
}
private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
{
var fromArg = GetFromArgs(attr.Arg);
if (fromArg != null)
{
return Assign(result, uniformProperty, fromArg);
}
var fromShort = GetFromArgs(attr.ArgShort);
if (fromShort != null)
{
return Assign(result, uniformProperty, fromShort);
}
return false;
}
private bool Assign(T result, PropertyInfo uniformProperty, object value)
{
if (uniformProperty.PropertyType == value.GetType())
{
uniformProperty.SetValue(result, value);
return true;
}
else
{
if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
{
uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
return true;
}
else
{
if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
throw new NotSupportedException();
}
}
}
private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
{
var s = value.ToString();
if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
{
uniformProperty.SetValue(result, e);
return true;
}
return false;
}
private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
{
if (int.TryParse(value.ToString(), out int i))
{
uniformProperty.SetValue(result, i);
return true;
}
return false;
}
private string? GetFromArgs(string key)
{
var argKey = $"--{key}=";
var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
if (arg != null)
{
return arg.Substring(argKey.Length);
}
return null;
}
}
}

13
ArgsUniform/ArgsUniform.csproj Normal file
View File

@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="I-Env" Version="1.0.0" />
</ItemGroup>
</Project>

29
ArgsUniform/ExampleUser.cs Normal file
View File

@ -0,0 +1,29 @@
namespace ArgsUniform
{
public class ExampleUser
{
public class Args
{
[Uniform("aaa", "a", "AAA", false, "Sets the AAA!")]
public string Aaa { get; set; } = string.Empty;
[Uniform("bbb", "b", "BBB", true, "Sets that BBB")]
public string Bbb { get; set; } = string.Empty;
}
public class DefaultsProvider
{
public string Aaa { get { return "non-static operation"; } }
}
public void Example()
{
// env var: "AAA=BBB"
var args = "--ccc=ddd";
var uniform = new ArgsUniform<Args>(new DefaultsProvider(), args);
var aaa = uniform.Parse();
}
}
}

20
ArgsUniform/UniformAttribute.cs Normal file
View File

@ -0,0 +1,20 @@
namespace ArgsUniform
{
public class UniformAttribute : Attribute
{
public UniformAttribute(string arg, string argShort, string envVar, bool required, string description)
{
Arg = arg;
ArgShort = argShort;
EnvVar = envVar;
Required = required;
Description = description;
}
public string Arg { get; }
public string ArgShort { get; }
public string EnvVar { get; }
public bool Required { get; }
public string Description { get; }
}
}

93
CodexNetDeployer/ArgOrVar.cs
View File

@ -1,93 +0,0 @@
namespace CodexNetDeployer
{
public class ArgOrVar
{
public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex.");
public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth.");
public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer.");
public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file.");
public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment.");
public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created.");
public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating.");
public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node.");
public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]");
private readonly string[] args;
public ArgOrVar(string[] args)
{
this.args = args;
}
public string Get(ArgVar key, string defaultValue = "")
{
var argKey = $"--{key.Arg}=";
var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
if (arg != null)
{
return arg.Substring(argKey.Length);
}
var env = Environment.GetEnvironmentVariable(key.Var);
if (env != null)
{
return env;
}
return defaultValue;
}
public int? GetInt(ArgVar key)
{
var str = Get(key);
if (string.IsNullOrEmpty(str)) return null;
if (int.TryParse(str, out int result))
{
return result;
}
return null;
}
public void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
"The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
"All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
Console.Write("\t[ CLI argument ] or [ Environment variable ]");
Console.CursorLeft = 70;
Console.Write("(Description)" + nl);
var fields = GetType().GetFields();
foreach (var field in fields)
{
var value = (ArgVar)field.GetValue(null)!;
value.PrintHelp();
}
}
}
public class ArgVar
{
public ArgVar(string arg, string var, string description)
{
Arg = arg;
Var = var;
Description = description;
}
public string Arg { get; }
public string Var { get; }
public string Description { get; }
public void PrintHelp()
{
Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]");
Console.CursorLeft = 70;
Console.Write(Description + Environment.NewLine);
}
}
}

1
CodexNetDeployer/CodexNetDeployer.csproj
View File

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
</ItemGroup>

82
CodexNetDeployer/Configuration.cs
View File

@ -1,49 +1,40 @@
using DistTestCore;
using ArgsUniform;
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
namespace CodexNetDeployer
{
public class Configuration
{
public Configuration(
string codexImage,
string gethImage,
string contractsImage,
string kubeConfigFile,
string kubeNamespace,
int? numberOfCodexNodes,
int? numberOfValidators,
int? storageQuota,
CodexLogLevel codexLogLevel,
TestRunnerLocation runnerLocation)
{
CodexImage = codexImage;
GethImage = gethImage;
ContractsImage = contractsImage;
KubeConfigFile = kubeConfigFile;
KubeNamespace = kubeNamespace;
NumberOfCodexNodes = numberOfCodexNodes;
NumberOfValidators = numberOfValidators;
StorageQuota = storageQuota;
CodexLogLevel = codexLogLevel;
RunnerLocation = runnerLocation;
}
[Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
public string CodexImage { get; }
public string GethImage { get; }
public string ContractsImage { get; }
public string KubeConfigFile { get; }
public string KubeNamespace { get; }
public int? NumberOfCodexNodes { get; }
public int? NumberOfValidators { get; }
public int? StorageQuota { get; }
public CodexLogLevel CodexLogLevel { get; }
public TestRunnerLocation RunnerLocation { get; }
[Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")]
public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
public void PrintConfig()
{
ForEachProperty(onString: Print, onInt: Print);
}
[Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")]
public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
[Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
[Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
public string KubeNamespace { get; set; } = string.Empty;
[Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
public int? NumberOfCodexNodes { get; set; }
[Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
public int? NumberOfValidators { get; set; }
[Uniform("storage-quota", "s", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
public int? StorageQuota { get; set; }
[Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
public List<string> Validate()
{
@ -75,7 +66,7 @@ namespace CodexNetDeployer
{
if (value == null || value.Value < 1)
{
errors.Add($"{variable} is must be set and must be greater than 0.");
errors.Add($"{variable} must be set and must be greater than 0.");
}
}
@ -83,19 +74,8 @@ namespace CodexNetDeployer
{
if (string.IsNullOrWhiteSpace(value))
{
errors.Add($"{variable} is must be set.");
errors.Add($"{variable} must be set.");
}
}
private static void Print(string variable, string value)
{
Console.WriteLine($"\t{variable}: '{value}'");
}
private static void Print(string variable, int? value)
{
if (value != null) Print(variable, value.ToString()!);
else Print(variable, "<NONE>");
}
}
}

17
CodexNetDeployer/Deployer.cs
View File

@ -1,6 +1,7 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
namespace CodexNetDeployer
{
@ -50,9 +51,11 @@ namespace CodexNetDeployer
private (WorkflowCreator, TestLifecycle) CreateFacilities()
{
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: config.KubeConfigFile,
kubeConfigFile: kubeConfig,
logPath: "null",
logDebug: false,
dataFilesPath: "notUsed",
@ -60,18 +63,24 @@ namespace CodexNetDeployer
runnerLocation: config.RunnerLocation
);
var kubeConfig = new KubernetesWorkflow.Configuration(
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: config.KubeNamespace,
kubeConfigFile: config.KubeConfigFile,
kubeConfigFile: kubeConfig,
operationTimeout: timeset.K8sOperationTimeout(),
retryDelay: timeset.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
return (workflowCreator, lifecycle);
}
private string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
private DeploymentMetadata CreateMetadata()
{
return new DeploymentMetadata(

51
CodexNetDeployer/Program.cs
View File

@ -1,9 +1,7 @@
using CodexNetDeployer;
using ArgsUniform;
using CodexNetDeployer;
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using Newtonsoft.Json;
using Utils;
using Configuration = CodexNetDeployer.Configuration;
public class Program
@ -11,46 +9,29 @@ public class Program
public static void Main(string[] args)
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDeployer" + nl + nl);
var argOrVar = new ArgOrVar(args);
Console.WriteLine("CodexNetDeployer" + nl);
if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
{
argOrVar.PrintHelp();
PrintHelp();
return;
}
var location = TestRunnerLocation.InternalToCluster;
var uniformArgs = new ArgsUniform<Configuration>(args);
var config = uniformArgs.Parse(true);
if (args.Any(a => a == "--external"))
{
location = TestRunnerLocation.ExternalToCluster;
config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
}
var config = new Configuration(
codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage),
gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage),
contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage),
kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile),
kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace),
numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes),
numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes),
storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota),
codexLogLevel: ParseEnum.Parse<CodexLogLevel>(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))),
runnerLocation: location
);
Console.WriteLine("Using:");
config.PrintConfig();
Console.WriteLine(nl);
var errors = config.Validate();
if (errors.Any())
{
Console.WriteLine($"Configuration errors: ({errors.Count})");
foreach ( var error in errors ) Console.WriteLine("\t" + error);
Console.WriteLine(nl);
argOrVar.PrintHelp();
PrintHelp();
return;
}
@ -63,4 +44,18 @@ public class Program
Console.WriteLine("Done!");
}
private static void PrintHelp()
{
var nl = Environment.NewLine;
Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
"The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
"All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
var uniformArgs = new ArgsUniform<Configuration>();
uniformArgs.PrintHelp();
}
}

49
ContinuousTests/AllTestsRun.cs
View File

@ -1,49 +0,0 @@
using Logging;
using Utils;
namespace ContinuousTests
{
public class AllTestsRun
{
private readonly Configuration config;
private readonly FixtureLog log;
private readonly TestFactory testFinder;
public AllTestsRun(Configuration config, FixtureLog log, TestFactory testFinder)
{
this.config = config;
this.log = log;
this.testFinder = testFinder;
}
public ContinuousTestResult RunAll()
{
var remainingTests = testFinder.CreateTests().ToList();
var result = ContinuousTestResult.Passed;
while (remainingTests.Any())
{
var test = remainingTests.PickOneRandom();
var testLog = log.CreateTestLog(test.Name);
var singleTestRun = new SingleTestRun(config, test, testLog);
log.Log($"Start '{test.Name}'");
try
{
singleTestRun.Run();
log.Log($"'{test.Name}' = Passed");
if (!config.KeepPassedTestLogs) testLog.Delete();
}
catch
{
log.Log($"'{test.Name}' = Failed");
testLog.MarkAsFailed();
result = ContinuousTestResult.Failed;
}
Thread.Sleep(config.SleepSecondsPerSingleTest * 1000);
}
return result;
}
}
}

11
ContinuousTests/CodexNodeFactory.cs
View File

@ -1,20 +1,17 @@
using DistTestCore;
using DistTestCore.Codex;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace ContinuousTests
{
public class CodexNodeFactory
{
public CodexNode[] Create(string[] urls, BaseLog log, ITimeSet timeSet)
public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
{
return urls.Select(url =>
return containers.Select(container =>
{
var cutIndex = url.LastIndexOf(':');
var host = url.Substring(0, cutIndex);
var port = url.Substring(cutIndex + 1);
var address = new Address(host, Convert.ToInt32(port));
var address = container.ClusterExternalAddress;
return new CodexNode(log, timeSet, address);
}).ToArray();
}

119
ContinuousTests/Configuration.cs
View File

@ -1,104 +1,47 @@
using Newtonsoft.Json;
using ArgsUniform;
using DistTestCore.Codex;
using Newtonsoft.Json;
namespace ContinuousTests
{
public class Configuration
{
public string LogPath { get; set; } = string.Empty;
public string[] CodexUrls { get; set; } = Array.Empty<string>();
public int SleepSecondsPerSingleTest { get; set; }
public int SleepSecondsPerAllTests { get; set; }
public bool KeepPassedTestLogs { get; set; }
[Uniform("log-path", "l", "LOGPATH", true, "Path where log files will be written.")]
public string LogPath { get; set; } = "logs";
[Uniform("data-path", "d", "DATAPATH", true, "Path where temporary data files will be written.")]
public string DataPath { get; set; } = "data";
[Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
public string CodexDeploymentJson { get; set; } = string.Empty;
[Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
public bool KeepPassedTestLogs { get; set; } = false;
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
public CodexDeployment CodexDeployment { get; set; } = null!;
}
public class ConfigLoader
{
private const string filename = "config.json";
public Configuration Load()
public Configuration Load(string[] args)
{
var config = Read();
var uniformArgs = new ArgsUniform<Configuration>(args);
Validate(config);
return config;
var result = uniformArgs.Parse(true);
result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
return result;
}
private Configuration Read()
private CodexDeployment ParseCodexDeploymentJson(string filename)
{
if (File.Exists(filename))
{
var lines = File.ReadAllText(filename);
try
{
var result = JsonConvert.DeserializeObject<Configuration>(lines);
if (result != null) return result;
}
catch { }
}
var logPath = Environment.GetEnvironmentVariable("LOGPATH");
var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS");
var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST");
var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS");
var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS");
if (!string.IsNullOrEmpty(logPath) &&
!string.IsNullOrEmpty(codexUrls) &&
!string.IsNullOrEmpty(sleepPerSingle) &&
!string.IsNullOrEmpty(sleepPerAll))
{
var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries);
int secondsSingle;
int secondsAll;
if (int.TryParse(sleepPerSingle, out secondsSingle) && int.TryParse(sleepPerAll, out secondsAll))
{
if (urls.Length > 0)
{
return new Configuration
{
LogPath = logPath,
CodexUrls = urls,
SleepSecondsPerSingleTest = secondsSingle,
SleepSecondsPerAllTests = secondsAll,
KeepPassedTestLogs = keep == "1"
};
}
}
}
var nl = Environment.NewLine;
throw new Exception($"Unable to load configuration from '{filename}', and " +
"unable to load configuration from environment variables. " + nl +
"'LOGPATH' = Path where log files will be saved." + nl +
"'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl +
"'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl +
"'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl +
"'KEEPPASSEDTESTLOGS' = (Optional, default: 0) Set to '1' to keep log files of tests that passed." + nl +
nl);
}
private void Validate(Configuration configuration)
{
if (configuration.SleepSecondsPerSingleTest < 1)
{
Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!");
configuration.SleepSecondsPerSingleTest = 1;
}
if (configuration.SleepSecondsPerAllTests < 1)
{
Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!");
configuration.SleepSecondsPerAllTests = 10;
}
if (string.IsNullOrEmpty(configuration.LogPath))
{
throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'");
}
if (!configuration.CodexUrls.Any())
{
throw new Exception("No Codex URLs found.");
}
var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
if (d == null) throw new Exception("Unable to parse " + filename);
return d;
}
}
}

22
ContinuousTests/ContinuousTest.cs
View File

@ -11,21 +11,33 @@ namespace ContinuousTests
public abstract class ContinuousTest
{
protected const int Zero = 0;
protected const int MinuteOne = 60;
protected const int MinuteFive = MinuteOne * 5;
protected const int HourOne = MinuteOne * 60;
protected const int HourThree = HourOne * 3;
protected const int DayOne = HourOne * 24;
protected const int DayThree = DayOne * 3;
private const string UploadFailedMessage = "Unable to store block";
public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager)
public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration)
{
Nodes = nodes;
Log = log;
FileManager = fileManager;
Configuration = configuration;
}
public CodexNode[] Nodes { get; private set; } = null!;
public BaseLog Log { get; private set; } = null!;
public IFileManager FileManager { get; private set; } = null!;
public Configuration Configuration { get; private set; } = null!;
public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
public abstract int RequiredNumberOfNodes { get; }
public abstract TimeSpan RunTestEvery { get; }
public abstract TestFailMode TestFailMode { get; }
public string Name
{
@ -35,8 +47,6 @@ namespace ContinuousTests
}
}
public abstract void Run();
public ContentId? UploadFile(CodexNode node, TestFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -79,4 +89,10 @@ namespace ContinuousTests
}
}
}
public enum TestFailMode
{
StopAfterFirstFailure,
AlwaysRunAllMoments
}
}

143
ContinuousTests/ContinuousTestRunner.cs
View File

@ -1,6 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using Logging;
using Logging;
namespace ContinuousTests
{
@ -8,130 +6,33 @@ namespace ContinuousTests
{
private readonly ConfigLoader configLoader = new ConfigLoader();
private readonly TestFactory testFactory = new TestFactory();
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly Configuration config;
private readonly StartupChecker startupChecker;
public ContinuousTestRunner(string[] args)
{
config = configLoader.Load(args);
startupChecker = new StartupChecker(config);
}
public void Run()
{
var config = //configLoader.Load();
new Configuration
{
CodexUrls =new[] { "http://localhost:8080", "http://localhost:8081" },
LogPath = "logs",
KeepPassedTestLogs = false,
SleepSecondsPerAllTests = 1,
SleepSecondsPerSingleTest = 1,
};
StartupChecks(config);
startupChecker.Check();
while (true)
var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
overviewLog.Log("Continuous tests starting...");
var allTests = testFactory.CreateTests();
var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
foreach (var t in testLoop)
{
var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun");
var allTestsRun = new AllTestsRun(config, log, testFactory);
var result = ContinuousTestResult.Passed;
try
{
result = allTestsRun.RunAll();
}
catch (Exception ex)
{
log.Error($"Exception during test run: " + ex);
}
if (result == ContinuousTestResult.Failed)
{
log.MarkAsFailed();
}
if (!config.KeepPassedTestLogs && result == ContinuousTestResult.Passed)
{
log.DeleteFolder();
}
Thread.Sleep(config.SleepSecondsPerSingleTest * 1000);
overviewLog.Log("Launching test-loop for " + t.Name);
t.Begin();
Thread.Sleep(TimeSpan.FromMinutes(5));
}
overviewLog.Log("All test-loops launched.");
while (true) Thread.Sleep((2 ^ 31) - 1);
}
private void StartupChecks(Configuration config)
{
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
log.Log("Starting continuous test run...");
log.Log("Checking configuration...");
PreflightCheck(config);
log.Log("Contacting Codex nodes...");
CheckCodexNodes(log, config);
log.Log("All OK.");
}
private void PreflightCheck(Configuration config)
{
var tests = testFactory.CreateTests();
if (!tests.Any())
{
throw new Exception("Unable to find any tests.");
}
var errors = new List<string>();
foreach (var test in tests)
{
if (test.RequiredNumberOfNodes > config.CodexUrls.Length)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Configuration only has {config.CodexUrls.Length}");
}
}
if (!Directory.Exists(config.LogPath))
{
Directory.CreateDirectory(config.LogPath);
}
if (errors.Any())
{
throw new Exception("Prerun check failed: " + string.Join(", ", errors));
}
}
private void CheckCodexNodes(BaseLog log, Configuration config)
{
var nodes = codexNodeFactory.Create(config.CodexUrls, log, new DefaultTimeSet());
var pass = true;
foreach (var n in nodes)
{
log.Log($"Checking '{n.Address.Host}'...");
if (EnsureOnline(n))
{
log.Log("OK");
}
else
{
log.Error($"No response from '{n.Address.Host}'.");
pass = false;
}
}
if (!pass)
{
throw new Exception("Not all codex nodes responded.");
}
}
private bool EnsureOnline(CodexNode n)
{
try
{
var info = n.GetDebugInfo();
if (info == null || string.IsNullOrEmpty(info.id)) return false;
}
catch
{
return false;
}
return true;
}
}
public enum ContinuousTestResult
{
Passed,
Failed
}
}

2
ContinuousTests/ContinuousTests.csproj
View File

@ -12,7 +12,9 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
<ProjectReference Include="..\Logging\Logging.csproj" />
</ItemGroup>

4
ContinuousTests/Program.cs
View File

@ -4,7 +4,9 @@ public class Program
{
public static void Main(string[] args)
{
var runner = new ContinuousTestRunner();
Console.WriteLine("Codex Continous-Test-Runner.");
Console.WriteLine("Running...");
var runner = new ContinuousTestRunner(args);
runner.Run();
}
}

170
ContinuousTests/SingleTestRun.cs
View File

@ -2,55 +2,189 @@
using DistTestCore;
using Logging;
using Utils;
using KubernetesWorkflow;
using NUnit.Framework.Internal;
using System.Reflection;
namespace ContinuousTests
{
public class SingleTestRun
{
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly List<Exception> exceptions = new List<Exception>();
private readonly Configuration config;
private readonly ContinuousTest test;
private readonly BaseLog overviewLog;
private readonly TestHandle handle;
private readonly CodexNode[] nodes;
private readonly FileManager fileManager;
private readonly FixtureLog fixtureLog;
private readonly string testName;
private readonly string dataFolder;
public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog)
public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle)
{
this.config = config;
this.test = test;
this.overviewLog = overviewLog;
this.handle = handle;
nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog);
fileManager = new FileManager(testLog, new DistTestCore.Configuration());
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
test.Initialize(nodes, testLog, fileManager);
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
}
public void Run()
{
test.Run();
Task.Run(() =>
{
try
{
RunTest();
fileManager.DeleteAllTestFiles();
Directory.Delete(dataFolder, true);
}
catch (Exception ex)
{
overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
Environment.Exit(-1);
}
});
}
public void TearDown()
private void RunTest()
{
test.Initialize(null!, null!, null!);
fileManager.DeleteAllTestFiles();
try
{
RunTestMoments();
if (!config.KeepPassedTestLogs) fixtureLog.Delete();
}
catch (Exception ex)
{
fixtureLog.Error("Test run failed with exception: " + ex);
fixtureLog.MarkAsFailed();
}
}
private CodexNode[] CreateRandomNodes(int number, BaseLog testLog)
private void RunTestMoments()
{
var urls = SelectRandomUrls(number);
testLog.Log("Selected nodes: " + string.Join(",", urls));
return codexNodeFactory.Create(urls, testLog, test.TimeSet);
var earliestMoment = handle.GetEarliestMoment();
var t = earliestMoment;
while (true)
{
RunMoment(t);
if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
{
Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
ThrowFailTest();
}
var nextMoment = handle.GetNextMoment(t);
if (nextMoment != null)
{
Log($" > Next TestMoment in {nextMoment.Value} seconds...");
t += nextMoment.Value;
Thread.Sleep(nextMoment.Value * 1000);
}
else
{
if (exceptions.Any())
{
ThrowFailTest();
}
OverviewLog(" > Test passed.");
return;
}
}
}
private string[] SelectRandomUrls(int number)
private void ThrowFailTest()
{
var urls = config.CodexUrls.ToList();
var result = new string[number];
var ex = UnpackException(exceptions.First());
Log(ex.ToString());
OverviewLog(" > Test failed: " + ex.Message);
throw ex;
}
private Exception UnpackException(Exception exception)
{
if (exception is AggregateException a)
{
return UnpackException(a.InnerExceptions.First());
}
if (exception is TargetInvocationException t)
{
return UnpackException(t.InnerException!);
}
return exception;
}
private void RunMoment(int t)
{
using (var context = new TestExecutionContext.IsolatedContext())
{
try
{
handle.InvokeMoment(t, InitializeTest);
}
catch (Exception ex)
{
exceptions.Add(ex);
}
}
DecommissionTest();
}
private void InitializeTest(string name)
{
Log($" > Running TestMoment '{name}'");
handle.Test.Initialize(nodes, fixtureLog, fileManager, config);
}
private void DecommissionTest()
{
handle.Test.Initialize(null!, null!, null!, null!);
}
private void Log(string msg)
{
fixtureLog.Log(msg);
}
private void OverviewLog(string msg)
{
Log(msg);
overviewLog.Log(testName + ": " + msg);
}
private CodexNode[] CreateRandomNodes(int number)
{
var containers = SelectRandomContainers(number);
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
}
private RunningContainer[] SelectRandomContainers(int number)
{
var containers = config.CodexDeployment.CodexContainers.ToList();
var result = new RunningContainer[number];
for (var i = 0; i < number; i++)
{
result[i] = urls.PickOneRandom();
result[i] = containers.PickOneRandom();
}
return result;
}
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
}
}
}

102
ContinuousTests/StartupChecker.cs Normal file
View File

@ -0,0 +1,102 @@
using DistTestCore.Codex;
using DistTestCore;
using Logging;
namespace ContinuousTests
{
public class StartupChecker
{
private readonly TestFactory testFactory = new TestFactory();
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly Configuration config;
public StartupChecker(Configuration config)
{
this.config = config;
}
public void Check()
{
var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
log.Log("Starting continuous test run...");
log.Log("Checking configuration...");
PreflightCheck(config);
log.Log("Contacting Codex nodes...");
CheckCodexNodes(log, config);
log.Log("All OK.");
}
private void PreflightCheck(Configuration config)
{
var tests = testFactory.CreateTests();
if (!tests.Any())
{
throw new Exception("Unable to find any tests.");
}
foreach (var test in tests)
{
var handle = new TestHandle(test);
handle.GetEarliestMoment();
handle.GetLastMoment();
}
var errors = new List<string>();
foreach (var test in tests)
{
if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
}
}
if (!Directory.Exists(config.LogPath))
{
Directory.CreateDirectory(config.LogPath);
}
if (errors.Any())
{
throw new Exception("Prerun check failed: " + string.Join(", ", errors));
}
}
private void CheckCodexNodes(BaseLog log, Configuration config)
{
var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
var pass = true;
foreach (var n in nodes)
{
log.Log($"Checking '{n.Address.Host}'...");
if (EnsureOnline(n))
{
log.Log("OK");
}
else
{
log.Error($"No response from '{n.Address.Host}'.");
pass = false;
}
}
if (!pass)
{
throw new Exception("Not all codex nodes responded.");
}
}
private bool EnsureOnline(CodexNode n)
{
try
{
var info = n.GetDebugInfo();
if (info == null || string.IsNullOrEmpty(info.id)) return false;
}
catch
{
return false;
}
return true;
}
}
}

86
ContinuousTests/TestHandle.cs Normal file
View File

@ -0,0 +1,86 @@
using System.Reflection;
namespace ContinuousTests
{
public class TestHandle
{
private readonly List<MethodMoment> moments = new List<MethodMoment>();
public TestHandle(ContinuousTest test)
{
Test = test;
ReflectTestMoments();
var testName = test.GetType().Name;
if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
}
public ContinuousTest Test { get; }
public int GetEarliestMoment()
{
return moments.Min(m => m.Moment);
}
public int GetLastMoment()
{
return moments.Max(m => m.Moment);
}
public int? GetNextMoment(int currentMoment)
{
var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
if (!remainingMoments.Any()) return null;
return remainingMoments.Min(m => m.Moment);
}
public void InvokeMoment(int currentMoment, Action<string> beforeInvoke)
{
var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
if (moment == null) return;
lock (MomentLock.Lock)
{
beforeInvoke(moment.Method.Name);
moment.Method.Invoke(Test, Array.Empty<object>());
}
}
private void ReflectTestMoments()
{
var methods = Test.GetType().GetMethods()
.Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0)
.ToArray();
foreach (var method in methods)
{
var moment = method.GetCustomAttribute<TestMomentAttribute>();
if (moment != null && moment.T >= 0)
{
moments.Add(new MethodMoment(method, moment.T));
}
}
}
}
public class MethodMoment
{
public MethodMoment(MethodInfo method, int moment)
{
Method = method;
Moment = moment;
if (moment < 0) throw new Exception("Moment must be zero or greater.");
}
public MethodInfo Method { get; }
public int Moment { get; }
}
public static class MomentLock
{
public static readonly object Lock = new();
}
}

52
ContinuousTests/TestLoop.cs Normal file
View File

@ -0,0 +1,52 @@
using Logging;
namespace ContinuousTests
{
public class TestLoop
{
private readonly Configuration config;
private readonly BaseLog overviewLog;
private readonly Type testType;
private readonly TimeSpan runsEvery;
public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
{
this.config = config;
this.overviewLog = overviewLog;
this.testType = testType;
this.runsEvery = runsEvery;
Name = testType.Name;
}
public string Name { get; }
public void Begin()
{
Task.Run(() =>
{
try
{
while (true)
{
StartTest();
Thread.Sleep(runsEvery);
}
}
catch(Exception ex)
{
overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
Environment.Exit(-1);
}
});
}
private void StartTest()
{
var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test);
var run = new SingleTestRun(config, overviewLog, handle);
run.Run();
}
}
}

12
ContinuousTests/TestMomentAttribute.cs Normal file
View File

@ -0,0 +1,12 @@
namespace ContinuousTests
{
public class TestMomentAttribute : Attribute
{
public TestMomentAttribute(int t)
{
T = t;
}
public int T { get; }
}
}

150
ContinuousTests/Tests/MarketplaceTest.cs Normal file
View File

@ -0,0 +1,150 @@
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
using Logging;
using Newtonsoft.Json;
using NUnit.Framework;
namespace ContinuousTests.Tests
{
public class MarketplaceTest : ContinuousTest
{
public override int RequiredNumberOfNodes => 1;
public override TimeSpan RunTestEvery => TimeSpan.FromDays(4);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
private readonly uint numberOfSlots = 3;
private readonly ByteSize fileSize = 10.MB();
private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
private TestFile file = null!;
private ContentId? cid;
private string purchaseId = string.Empty;
[TestMoment(t: Zero)]
public void NodePostsStorageRequest()
{
var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
Log.Log("expected total cost: " + expectedTotalCost);
file = FileManager.GenerateTestFile(fileSize);
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
try
{
var debugInfo = Nodes[0].GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
interaction.MintTestTokens(new[] { account.Account }, expectedTotalCost, tokenAddress);
var container = rc.Containers[0];
var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork;
var codexAccess = new CodexAccess(lifecycle, container);
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
cid = UploadFile(codexAccess.Node, file);
Assert.That(cid, Is.Not.Null);
var balance = marketAccess.GetBalance();
Log.Log("Account: " + account.Account);
Log.Log("Balance: " + balance);
purchaseId = marketAccess.RequestStorage(
contentId: cid!,
pricePerSlotPerSecond: pricePerSlotPerSecond,
requiredCollateral: 100.TestTokens(),
minRequiredNumberOfNodes: numberOfSlots,
proofProbability: 10,
duration: contractDuration);
Log.Log($"PurchaseId: '{purchaseId}'");
Assert.That(!string.IsNullOrEmpty(purchaseId));
}
finally
{
flow.DeleteTestResources();
}
}
[TestMoment(t: DayThree)]
public void StoredDataIsAvailableAfterThreeDays()
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
try
{
var debugInfo = Nodes[0].GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var container = rc.Containers[0];
var codexAccess = new CodexAccess(lifecycle, container);
var result = DownloadContent(codexAccess.Node, cid!);
file.AssertIsEqual(result);
}
finally
{
flow.DeleteTestResources();
}
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
{
var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
(
kubeConfigFile: kubeConfig,
logPath: "null",
logDebug: false,
dataFilesPath: Configuration.LogPath,
codexLogLevel: CodexLogLevel.Debug,
runnerLocation: TestRunnerLocation.ExternalToCluster
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: MarketplaceTestNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: TimeSet.K8sOperationTimeout(),
retryDelay: TimeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
return (workflowCreator, lifecycle);
}
private string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
}
}

12
ContinuousTests/Tests/PerformanceTests.cs
View File

@ -8,7 +8,8 @@ namespace ContinuousTests.Tests
{
public override int RequiredNumberOfNodes => 1;
public override void Run()
[TestMoment(t: Zero)]
public void UploadTest()
{
UploadTest(100, Nodes[0]);
}
@ -18,7 +19,8 @@ namespace ContinuousTests.Tests
{
public override int RequiredNumberOfNodes => 1;
public override void Run()
[TestMoment(t: Zero)]
public void DownloadTest()
{
DownloadTest(100, Nodes[0], Nodes[0]);
}
@ -28,7 +30,8 @@ namespace ContinuousTests.Tests
{
public override int RequiredNumberOfNodes => 2;
public override void Run()
[TestMoment(t: Zero)]
public void DownloadTest()
{
DownloadTest(100, Nodes[0], Nodes[1]);
}
@ -36,6 +39,9 @@ namespace ContinuousTests.Tests
public abstract class PerformanceTest : ContinuousTest
{
public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
public void UploadTest(int megabytes, CodexNode uploadNode)
{
var file = FileManager.GenerateTestFile(megabytes.MB());

16
ContinuousTests/Tests/TwoClientTest.cs
View File

@ -6,14 +6,24 @@ namespace ContinuousTests.Tests
public class TwoClientTest : ContinuousTest
{
public override int RequiredNumberOfNodes => 2;
public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
public override void Run()
private ContentId? cid;
private TestFile file = null!;
[TestMoment(t: Zero)]
public void UploadTestFile()
{
var file = FileManager.GenerateTestFile(10.MB());
file = FileManager.GenerateTestFile(10.MB());
var cid = UploadFile(Nodes[0], file);
cid = UploadFile(Nodes[0], file);
Assert.That(cid, Is.Not.Null);
}
[TestMoment(t: MinuteFive)]
public void DownloadTestFile()
{
var dl = DownloadContent(Nodes[1], cid!);
file.AssertIsEqual(dl);

11
DistTestCore/Codex/CodexNode.cs
View File

@ -65,6 +65,11 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
{
return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
@ -170,4 +175,10 @@ namespace DistTestCore.Codex
public uint? nodes { get; set; }
public uint? tolerance { get; set; }
}
public class CodexStoragePurchase
{
public string state { get; set; } = string.Empty;
public string error { get; set; } = string.Empty;
}
}

2
DistTestCore/DistTest.cs
View File

@ -250,7 +250,7 @@ namespace DistTestCore
{
OnEachCodexNode(lifecycle, node =>
{
lifecycle.DownloadLog(node);
lifecycle.DownloadLog(node.CodexAccess.Container);
});
}

10
DistTestCore/Logs/CodexNodeLog.cs → DistTestCore/Logs/DownloadedLog.cs
View File

@ -3,17 +3,17 @@ using NUnit.Framework;
namespace DistTestCore.Logs
{
public interface ICodexNodeLog
public interface IDownloadedLog
{
void AssertLogContains(string expectedString);
}
public class CodexNodeLog : ICodexNodeLog
public class DownloadedLog : IDownloadedLog
{
private readonly LogFile logFile;
private readonly OnlineCodexNode owner;
private readonly string owner;
public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
public DownloadedLog(LogFile logFile, string owner)
{
this.logFile = logFile;
this.owner = owner;
@ -31,7 +31,7 @@ namespace DistTestCore.Logs
line = streamReader.ReadLine();
}
Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
}
}
}

10
DistTestCore/Logs/LogDownloadHandler.cs
View File

@ -5,21 +5,21 @@ namespace DistTestCore.Logs
{
public class LogDownloadHandler : LogHandler, ILogHandler
{
private readonly OnlineCodexNode node;
private readonly RunningContainer container;
private readonly LogFile log;
public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
public LogDownloadHandler(RunningContainer container, string description, LogFile log)
{
this.node = node;
this.container = container;
this.log = log;
log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description);
}
public CodexNodeLog CreateCodexNodeLog()
public DownloadedLog DownloadLog()
{
return new CodexNodeLog(log, node);
return new DownloadedLog(log, container.Name);
}
protected override void ProcessLine(string line)

5
DistTestCore/Marketplace/GethStartResult.cs
View File

@ -1,4 +1,6 @@
namespace DistTestCore.Marketplace
using Newtonsoft.Json;
namespace DistTestCore.Marketplace
{
public class GethStartResult
{
@ -9,6 +11,7 @@
CompanionNode = companionNode;
}
[JsonIgnore]
public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
public MarketplaceNetwork MarketplaceNetwork { get; }
public GethCompanionNodeInfo CompanionNode { get; }

8
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
public interface IMarketplaceAccess
{
string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
void AssertThatBalance(IResolveConstraint constraint, string message = "");
TestToken GetBalance();
}
@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace
this.codexAccess = codexAccess;
}
public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
{
var request = new CodexSalesRequestStorageRequest
{
duration = ToHexBigInt(duration.TotalSeconds),
proofProbability = ToHexBigInt(proofProbability),
reward = ToHexBigInt(pricePerBytePerSecond),
reward = ToHexBigInt(pricePerSlotPerSecond),
collateral = ToHexBigInt(requiredCollateral),
expiry = null,
nodes = minRequiredNumberOfNodes,
@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
};
Log($"Requesting storage for: {contentId.Id}... (" +
$"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
$"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
$"requiredCollateral: {requiredCollateral}, " +
$"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
$"proofProbability: {proofProbability}, " +

6
DistTestCore/OnlineCodexNode.cs
View File

@ -16,7 +16,7 @@ namespace DistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
ICodexNodeLog DownloadLog();
IDownloadedLog DownloadLog();
IMetricsAccess Metrics { get; }
IMarketplaceAccess Marketplace { get; }
ICodexSetup BringOffline();
@ -107,9 +107,9 @@ namespace DistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public ICodexNodeLog DownloadLog()
public IDownloadedLog DownloadLog()
{
return lifecycle.DownloadLog(this);
return lifecycle.DownloadLog(CodexAccess.Container);
}
public ICodexSetup BringOffline()

10
DistTestCore/TestLifecycle.cs
View File

@ -41,16 +41,16 @@ namespace DistTestCore
FileManager.DeleteAllTestFiles();
}
public ICodexNodeLog DownloadLog(OnlineCodexNode node)
public IDownloadedLog DownloadLog(RunningContainer container)
{
var subFile = Log.CreateSubfile();
var description = node.GetName();
var handler = new LogDownloadHandler(node, description, subFile);
var description = container.Name;
var handler = new LogDownloadHandler(container, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
CodexStarter.DownloadLog(container, handler);
return new CodexNodeLog(subFile, node);
return new DownloadedLog(subFile, description);
}
public string GetTestDuration()

11
KubernetesWorkflow/K8sController.cs
View File

@ -39,7 +39,7 @@ namespace KubernetesWorkflow
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var podInfo = FetchNewPod();
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
}
public void Stop(RunningPod pod)
@ -436,9 +436,9 @@ namespace KubernetesWorkflow
#region Service management
private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
{
var result = new Dictionary<ContainerRecipe, Port[]>();
var result = new List<ContainerRecipePortMapEntry>();
var ports = CreateServicePorts(containerRecipes);
@ -468,7 +468,7 @@ namespace KubernetesWorkflow
return (serviceSpec.Metadata.Name, result);
}
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
{
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@ -485,7 +485,8 @@ namespace KubernetesWorkflow
// These service ports belongs to this recipe.
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
result.Add(r, ports);
result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
}
}
}

16
KubernetesWorkflow/RunningContainers.cs
View File

@ -23,12 +23,12 @@ namespace KubernetesWorkflow
public class RunningContainer
{
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, Address clusterExternalAddress, Address clusterInternalAddress)
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
{
Pod = pod;
Recipe = recipe;
ServicePorts = servicePorts;
Name = GetContainerName(recipe, startupConfig);
Name = name;
ClusterExternalAddress = clusterExternalAddress;
ClusterInternalAddress = clusterInternalAddress;
}
@ -39,17 +39,5 @@ namespace KubernetesWorkflow
public Port[] ServicePorts { get; }
public Address ClusterExternalAddress { get; }
public Address ClusterInternalAddress { get; }
private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
{
if (!string.IsNullOrEmpty(startupConfig.NameOverride))
{
return $"<{startupConfig.NameOverride}{recipe.Number}>";
}
else
{
return $"<{recipe.Name}>";
}
}
}
}

33
KubernetesWorkflow/RunningPod.cs
View File

@ -2,35 +2,50 @@
{
public class RunningPod
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, ContainerRecipePortMapEntry[] portMapEntries)
{
Cluster = cluster;
PodInfo = podInfo;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
PortMapEntries = portMapEntries;
}
public K8sCluster Cluster { get; }
public PodInfo PodInfo { get; }
public ContainerRecipePortMapEntry[] PortMapEntries { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{
if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
return servicePortMap[containerRecipe];
if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
{
return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
}
return Array.Empty<Port>();
}
}
public class ContainerRecipePortMapEntry
{
public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
{
ContainerNumber = containerNumber;
Ports = ports;
}
public int ContainerNumber { get; }
public Port[] Ports { get; }
}
public class PodInfo
{
public PodInfo(string podName, string podIp, string k8sNodeName)
public PodInfo(string name, string ip, string k8sNodeName)
{
Name = podName;
Ip = podIp;
Name = name;
Ip = ip;
K8SNodeName = k8sNodeName;
}

17
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -81,13 +81,28 @@ namespace KubernetesWorkflow
var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
return new RunningContainer(runningPod, r, servicePorts, startupConfig,
var name = GetContainerName(r, startupConfig);
return new RunningContainer(runningPod, r, servicePorts, name,
GetContainerExternalAddress(runningPod, servicePorts),
GetContainerInternalAddress(r));
}).ToArray();
}
private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
{
if (startupConfig == null) return "";
if (!string.IsNullOrEmpty(startupConfig.NameOverride))
{
return $"<{startupConfig.NameOverride}{recipe.Number}>";
}
else
{
return $"<{recipe.Name}>";
}
}
private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
{
return new Address(

6
CodexNetDeployer/NullLog.cs → Logging/NullLog.cs
View File

@ -1,6 +1,4 @@
using Logging;
namespace CodexNetDeployer
namespace Logging
{
public class NullLog : TestLog
{
@ -15,12 +13,10 @@ namespace CodexNetDeployer
public override void Log(string message)
{
//Console.WriteLine(message);
}
public override void Debug(string message = "", int skipFrames = 0)
{
//Console.WriteLine(message);
}
public override void Error(string message)

2
Tests/BasicTests/ExampleTests.cs
View File

@ -64,7 +64,7 @@ namespace Tests.BasicTests
var contentId = buyer.UploadFile(testFile);
buyer.Marketplace.RequestStorage(contentId,
pricePerBytePerSecond: 2.TestTokens(),
pricePerSlotPerSecond: 2.TestTokens(),
requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1,
proofProbability: 5,

8
cs-codex-dist-testing.sln
View File

@ -19,7 +19,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nether
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -63,6 +65,10 @@ Global
{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE