Merge branch 'master' into feature/codex-net-deployer

ArgsUniform/ArgsUniform.cs

@@ -0,0 +1,235 @@
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class ArgsUniform<T>
+    {
+        private readonly object? defaultsProvider;
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private const int cliStart = 8;
+        private const int shortStart = 38;
+        private const int envStart = 48;
+        private const int descStart = 80;
+
+        public ArgsUniform(params string[] args)
+            : this(new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(object defaultsProvider, params string[] args)
+            : this(defaultsProvider, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(IEnv.IEnv env, params string[] args)
+            : this(null!, env, args)
+        {
+        }
+
+        public ArgsUniform(object defaultsProvider, IEnv.IEnv env, params string[] args)
+        {
+            this.defaultsProvider = defaultsProvider;
+            this.env = env;
+            this.args = args;
+        }
+
+        public T Parse(bool printResult = false)
+        {
+            var result = Activator.CreateInstance<T>();
+            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            var missingRequired = new List<PropertyInfo>();
+            foreach (var uniformProperty in uniformProperties)
+            {
+                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
+                if (attr != null)
+                {
+                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    {
+                        {
+                            missingRequired.Add(uniformProperty);
+                        }
+                    }
+                }
+            }
+
+            if (missingRequired.Any())
+            {
+                PrintResults(result, uniformProperties);
+                Print("");
+                foreach (var missing in missingRequired)
+                {
+                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
+                    var exampleArg = $"--{attr.Arg}=...";
+                    var exampleEnvVar = $"{attr.EnvVar}=...";
+                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
+                }
+
+                PrintHelp();
+                throw new ArgumentException("Unable to assemble all required arguments");
+            }
+
+            if (printResult)
+            {
+                PrintResults(result, uniformProperties);
+            }
+
+            return result;
+        }
+
+        private void PrintResults(T result, PropertyInfo[] uniformProperties)
+        {
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
+        public void PrintHelp()
+        {
+            Print("");
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
+            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
+            foreach (var attr in attrs)
+            {
+                var a = attr!;
+                var optional = !a.Required ? " *" : "";
+                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+            }
+            Print("");
+        }
+
+        private void Print(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+
+        private void PrintAligned(string cli, string s, string env, string desc)
+        {
+            Console.CursorLeft = cliStart;
+            Console.Write(cli);
+            Console.CursorLeft = shortStart;
+            Console.Write(s);
+            Console.CursorLeft = envStart;
+            Console.Write(env);
+            Console.CursorLeft = descStart;
+            Console.Write(desc + Environment.NewLine);
+        }
+
+        private object GetDefaultValue(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+
+        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var currentValue = uniformProperty.GetValue(result);
+            var isEmptryString = (currentValue as string) == string.Empty;
+            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
+
+            if (defaultsProvider == null) return false;
+
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return false;
+
+            var value = defaultProperty.GetValue(defaultsProvider);
+            if (value != null)
+            {
+                return Assign(result, uniformProperty, value);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    
+                    throw new NotSupportedException();
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+    }
+}

ArgsUniform/ArgsUniform.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="I-Env" Version="1.0.0" />
+  </ItemGroup>
+
+</Project>

ArgsUniform/ExampleUser.cs

@@ -0,0 +1,29 @@
+namespace ArgsUniform
+{
+    public class ExampleUser
+    {
+        public class Args
+        {
+            [Uniform("aaa", "a", "AAA", false, "Sets the AAA!")]
+            public string Aaa { get; set; } = string.Empty;
+
+            [Uniform("bbb", "b", "BBB", true, "Sets that BBB")]
+            public string Bbb { get; set; } = string.Empty;
+        }
+
+        public class DefaultsProvider
+        {
+            public string Aaa { get { return "non-static operation"; } }
+        }
+
+        public void Example()
+        {
+            // env var: "AAA=BBB"
+            var args = "--ccc=ddd";
+
+            var uniform = new ArgsUniform<Args>(new DefaultsProvider(), args);
+
+            var aaa = uniform.Parse();
+        }
+    }
+}

ArgsUniform/UniformAttribute.cs

@@ -0,0 +1,20 @@
+namespace ArgsUniform
+{
+    public class UniformAttribute : Attribute
+    {
+        public UniformAttribute(string arg, string argShort, string envVar, bool required, string description)
+        {
+            Arg = arg;
+            ArgShort = argShort;
+            EnvVar = envVar;
+            Required = required;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string ArgShort { get; }
+        public string EnvVar { get; }
+        public bool Required { get; }
+        public string Description { get; }
+    }
+}

CodexNetDeployer/ArgOrVar.cs

@@ -1,93 +0,0 @@
-namespace CodexNetDeployer
-{
-    public class ArgOrVar
-    {
-        public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex.");
-        public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth.");
-        public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer.");
-        public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file.");
-        public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment.");
-        public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created.");
-        public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating.");
-        public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node.");
-        public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]");
-
-        private readonly string[] args;
-
-        public ArgOrVar(string[] args)
-        {
-            this.args = args;
-        }
-
-        public string Get(ArgVar key, string defaultValue = "")
-        {
-            var argKey = $"--{key.Arg}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-
-            var env = Environment.GetEnvironmentVariable(key.Var);
-            if (env != null)
-            {
-                return env;
-            }
-
-            return defaultValue;
-        }
-        
-        public int? GetInt(ArgVar key)
-        {
-            var str = Get(key);
-            if (string.IsNullOrEmpty(str)) return null;
-            if (int.TryParse(str, out int result))
-            {
-                return result;
-            }
-            return null;
-        }
-
-        public void PrintHelp()
-        {
-            var nl = Environment.NewLine;
-            Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
-                "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
-                "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
-
-            Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
-                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
-
-            Console.Write("\t[ CLI argument ] or [ Environment variable ]");
-            Console.CursorLeft = 70;
-            Console.Write("(Description)" + nl);
-            var fields = GetType().GetFields();
-            foreach (var field in fields)
-            {
-                var value = (ArgVar)field.GetValue(null)!;
-                value.PrintHelp();
-            }
-        }
-    }
-
-    public class ArgVar
-    {
-        public ArgVar(string arg, string var, string description)
-        {
-            Arg = arg;
-            Var = var;
-            Description = description;
-        }
-
-        public string Arg { get; }
-        public string Var { get; }
-        public string Description { get; }
-
-        public void PrintHelp()
-        {
-            Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]");
-            Console.CursorLeft = 70;
-            Console.Write(Description + Environment.NewLine);
-        }
-    }
-}

CodexNetDeployer/CodexNetDeployer.csproj

@@ -8,6 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

CodexNetDeployer/Configuration.cs

@@ -1,49 +1,40 @@
-using DistTestCore;
+using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Marketplace;
 
 namespace CodexNetDeployer
 {
     public class Configuration
     {
-        public Configuration(
+        [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
-            string codexImage,
+        public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
-            string gethImage,
-            string contractsImage,
-            string kubeConfigFile,
-            string kubeNamespace,
-            int? numberOfCodexNodes,
-            int? numberOfValidators,
-            int? storageQuota,
-            CodexLogLevel codexLogLevel,
-            TestRunnerLocation runnerLocation)
-        {
-            CodexImage = codexImage;
-            GethImage = gethImage;
-            ContractsImage = contractsImage;
-            KubeConfigFile = kubeConfigFile;
-            KubeNamespace = kubeNamespace;
-            NumberOfCodexNodes = numberOfCodexNodes;
-            NumberOfValidators = numberOfValidators;
-            StorageQuota = storageQuota;
-            CodexLogLevel = codexLogLevel;
-            RunnerLocation = runnerLocation;
-        }
 
-        public string CodexImage { get; }
+        [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")]
-        public string GethImage { get; }
+        public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
-        public string ContractsImage { get; }
-        public string KubeConfigFile { get; }
-        public string KubeNamespace { get; }
-        public int? NumberOfCodexNodes { get; }
-        public int? NumberOfValidators { get; }
-        public int? StorageQuota { get; }
-        public CodexLogLevel CodexLogLevel { get; }
-        public TestRunnerLocation RunnerLocation { get; }
 
-        public void PrintConfig()
+        [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")]
-        {
+        public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
-            ForEachProperty(onString: Print, onInt: Print);
+
-        }
+        [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
+        public string KubeNamespace { get; set; } = string.Empty;
+
+        [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
+        public int? NumberOfCodexNodes { get; set; }
+
+        [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
+        public int? NumberOfValidators { get; set; }
+
+        [Uniform("storage-quota", "s", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
+        public int? StorageQuota { get; set; }
+
+        [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
+        public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
 
         public List<string> Validate()
         {
@@ -75,7 +66,7 @@ namespace CodexNetDeployer
         {
             if (value == null || value.Value < 1)
             {
-                errors.Add($"{variable} is must be set and must be greater than 0.");
+                errors.Add($"{variable} must be set and must be greater than 0.");
             }
         }
 
@@ -83,19 +74,8 @@ namespace CodexNetDeployer
         {
             if (string.IsNullOrWhiteSpace(value))
             {
-                errors.Add($"{variable} is must be set.");
+                errors.Add($"{variable} must be set.");
             }
         }
-
-        private static void Print(string variable, string value)
-        {
-            Console.WriteLine($"\t{variable}: '{value}'");
-        }
-
-        private static void Print(string variable, int? value)
-        {
-            if (value != null) Print(variable, value.ToString()!);
-            else Print(variable, "<NONE>");
-        }
     }
 }

CodexNetDeployer/Deployer.cs

@@ -1,6 +1,7 @@
 using DistTestCore;
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using Logging;
 
 namespace CodexNetDeployer
 {
@@ -50,9 +51,11 @@ namespace CodexNetDeployer
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
+            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+
             var lifecycleConfig = new DistTestCore.Configuration
             (
-                kubeConfigFile: config.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 logPath: "null",
                 logDebug: false,
                 dataFilesPath: "notUsed",
@@ -60,18 +63,24 @@ namespace CodexNetDeployer
                 runnerLocation: config.RunnerLocation
             );
 
-            var kubeConfig = new KubernetesWorkflow.Configuration(
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: config.KubeNamespace,
-                kubeConfigFile: config.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 operationTimeout: timeset.K8sOperationTimeout(),
                 retryDelay: timeset.WaitForK8sServiceDelay());
 
-            var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
             var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
 
             return (workflowCreator, lifecycle);
         }
 
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(

CodexNetDeployer/Program.cs

@@ -1,9 +1,7 @@
-using CodexNetDeployer;
+using ArgsUniform;
+using CodexNetDeployer;
 using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
 using Newtonsoft.Json;
-using Utils;
 using Configuration = CodexNetDeployer.Configuration;
 
 public class Program
@@ -11,46 +9,29 @@ public class Program
     public static void Main(string[] args)
     {
         var nl = Environment.NewLine;
-        Console.WriteLine("CodexNetDeployer" + nl + nl);
+        Console.WriteLine("CodexNetDeployer" + nl);
-
-        var argOrVar = new ArgOrVar(args);
 
         if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
         {
-            argOrVar.PrintHelp();
+            PrintHelp();
             return;
         }
 
-        var location = TestRunnerLocation.InternalToCluster;
+        var uniformArgs = new ArgsUniform<Configuration>(args);
+        var config = uniformArgs.Parse(true);
+        
         if (args.Any(a => a == "--external"))
         {
-            location = TestRunnerLocation.ExternalToCluster;
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
         }
 
-        var config = new Configuration(
-            codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage),
-            gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage),
-            contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage),
-            kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile),
-            kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace),
-            numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes),
-            numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes),
-            storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota),
-            codexLogLevel: ParseEnum.Parse<CodexLogLevel>(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))),
-            runnerLocation: location
-        );
-
-        Console.WriteLine("Using:");
-        config.PrintConfig();
-        Console.WriteLine(nl);
-
         var errors = config.Validate();
         if (errors.Any())
         {
             Console.WriteLine($"Configuration errors: ({errors.Count})");
             foreach ( var error in errors ) Console.WriteLine("\t" + error);
             Console.WriteLine(nl);
-            argOrVar.PrintHelp();
+            PrintHelp();
             return;
         }
 
@@ -63,4 +44,18 @@ public class Program
 
         Console.WriteLine("Done!");
     }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+            "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
+            "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+        Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+
+        var uniformArgs = new ArgsUniform<Configuration>();
+        uniformArgs.PrintHelp();
+    }
 }

ContinuousTests/AllTestsRun.cs

@@ -1,49 +0,0 @@
-using Logging;
-using Utils;
-
-namespace ContinuousTests
-{
-    public class AllTestsRun
-    {
-        private readonly Configuration config;
-        private readonly FixtureLog log;
-        private readonly TestFactory testFinder;
-
-        public AllTestsRun(Configuration config, FixtureLog log, TestFactory testFinder)
-        {
-            this.config = config;
-            this.log = log;
-            this.testFinder = testFinder;
-        }
-
-        public ContinuousTestResult RunAll()
-        {
-            var remainingTests = testFinder.CreateTests().ToList();
-            var result = ContinuousTestResult.Passed;
-            while (remainingTests.Any())
-            {
-                var test = remainingTests.PickOneRandom();
-                var testLog = log.CreateTestLog(test.Name);
-                var singleTestRun = new SingleTestRun(config, test, testLog);
-
-                log.Log($"Start '{test.Name}'");
-                try
-                {
-                    singleTestRun.Run();
-                    log.Log($"'{test.Name}' = Passed");
-                    if (!config.KeepPassedTestLogs) testLog.Delete();
-                }
-                catch
-                {
-                    log.Log($"'{test.Name}' = Failed");
-                    testLog.MarkAsFailed();
-                    result = ContinuousTestResult.Failed;
-                }
-                
-                Thread.Sleep(config.SleepSecondsPerSingleTest * 1000);
-            }
-
-            return result;
-        }
-    }
-}

ContinuousTests/CodexNodeFactory.cs

@@ -1,20 +1,17 @@
 using DistTestCore;
 using DistTestCore.Codex;
+using KubernetesWorkflow;
 using Logging;
-using Utils;
 
 namespace ContinuousTests
 {
     public class CodexNodeFactory
     {
-        public CodexNode[] Create(string[] urls, BaseLog log, ITimeSet timeSet)
+        public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
         {
-            return urls.Select(url =>
+            return containers.Select(container =>
             {
-                var cutIndex = url.LastIndexOf(':');
+                var address = container.ClusterExternalAddress;
-                var host = url.Substring(0, cutIndex);
-                var port = url.Substring(cutIndex + 1);
-                var address = new Address(host, Convert.ToInt32(port));
                 return new CodexNode(log, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -1,104 +1,47 @@
-using Newtonsoft.Json;
+using ArgsUniform;
+using DistTestCore.Codex;
+using Newtonsoft.Json;
 
 namespace ContinuousTests
 {
     public class Configuration
     {
-        public string LogPath { get; set; } = string.Empty;
+        [Uniform("log-path", "l", "LOGPATH", true, "Path where log files will be written.")]
-        public string[] CodexUrls { get; set; } = Array.Empty<string>();
+        public string LogPath { get; set; } = "logs";
-        public int SleepSecondsPerSingleTest { get; set; }
+
-        public int SleepSecondsPerAllTests { get; set; }
+        [Uniform("data-path", "d", "DATAPATH", true, "Path where temporary data files will be written.")]
-        public bool KeepPassedTestLogs { get; set; }
+        public string DataPath { get; set; } = "data";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
+        public bool KeepPassedTestLogs { get; set; } = false;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
     }
 
     public class ConfigLoader
     {
-        private const string filename = "config.json";
+        public Configuration Load(string[] args)
-
-        public Configuration Load()
         {
-            var config = Read();
+            var uniformArgs = new ArgsUniform<Configuration>(args);
 
-            Validate(config);
+            var result = uniformArgs.Parse(true);
-            return config;
+            
+            result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
+
+            return result;
         }
-
+        
-        private Configuration Read()
+        private CodexDeployment ParseCodexDeploymentJson(string filename)
         {
-            if (File.Exists(filename))
+            var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
-            {
+            if (d == null) throw new Exception("Unable to parse " + filename);
-                var lines = File.ReadAllText(filename);
+            return d;
-                try
-                {
-                    var result = JsonConvert.DeserializeObject<Configuration>(lines);
-                    if (result != null) return result;
-                }
-                catch { }
-            }
-
-            var logPath = Environment.GetEnvironmentVariable("LOGPATH");
-            var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS");
-            var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST");
-            var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS");
-            var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS");
-
-            if (!string.IsNullOrEmpty(logPath) &&
-                !string.IsNullOrEmpty(codexUrls) &&
-                !string.IsNullOrEmpty(sleepPerSingle) &&
-                !string.IsNullOrEmpty(sleepPerAll))
-            {
-                var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries);
-                int secondsSingle;
-                int secondsAll;
-                if (int.TryParse(sleepPerSingle, out secondsSingle) && int.TryParse(sleepPerAll, out secondsAll))
-                {
-                    if (urls.Length > 0)
-                    {
-                        return new Configuration
-                        { 
-                            LogPath = logPath,
-                            CodexUrls = urls,
-                            SleepSecondsPerSingleTest = secondsSingle,
-                            SleepSecondsPerAllTests = secondsAll,
-                            KeepPassedTestLogs = keep == "1"
-                        };
-                    }
-                }
-            }
-
-            var nl = Environment.NewLine;
-            throw new Exception($"Unable to load configuration from '{filename}', and " +
-                "unable to load configuration from environment variables. " + nl +
-                "'LOGPATH' = Path where log files will be saved." + nl +
-                "'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl +
-                "'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl +
-                "'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl +
-                "'KEEPPASSEDTESTLOGS' = (Optional, default: 0) Set to '1' to keep log files of tests that passed." + nl +
-                nl);
-        }
-
-        private void Validate(Configuration configuration)
-        {
-            if (configuration.SleepSecondsPerSingleTest < 1)
-            {
-                Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!");
-                configuration.SleepSecondsPerSingleTest = 1;
-            }
-            if (configuration.SleepSecondsPerAllTests < 1)
-            {
-                Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!");
-                configuration.SleepSecondsPerAllTests = 10;
-            }
-
-            if (string.IsNullOrEmpty(configuration.LogPath))
-            {
-                throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'");
-            }
-
-            if (!configuration.CodexUrls.Any())
-            {
-                throw new Exception("No Codex URLs found.");
-            }
         }
     }
 }

ContinuousTests/ContinuousTest.cs

@@ -11,21 +11,33 @@ namespace ContinuousTests
 
     public abstract class ContinuousTest
     {
+        protected const int Zero = 0;
+        protected const int MinuteOne = 60;
+        protected const int MinuteFive = MinuteOne * 5;
+        protected const int HourOne = MinuteOne * 60;
+        protected const int HourThree = HourOne * 3;
+        protected const int DayOne = HourOne * 24;
+        protected const int DayThree = DayOne * 3;
+
         private const string UploadFailedMessage = "Unable to store block";
 
-        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager)
+        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration)
         {
             Nodes = nodes;
             Log = log;
             FileManager = fileManager;
+            Configuration = configuration;
         }
 
         public CodexNode[] Nodes { get; private set; } = null!;
         public BaseLog Log { get; private set; } = null!;
         public IFileManager FileManager { get; private set; } = null!;
+        public Configuration Configuration { get; private set; } = null!;
         public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
 
         public abstract int RequiredNumberOfNodes { get; }
+        public abstract TimeSpan RunTestEvery { get; }
+        public abstract TestFailMode TestFailMode { get; }
 
         public string Name
         {
@@ -35,8 +47,6 @@ namespace ContinuousTests
             }
         }
 
-        public abstract void Run();
-
         public ContentId? UploadFile(CodexNode node, TestFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
@@ -79,4 +89,10 @@ namespace ContinuousTests
             }
         }
     }
+
+    public enum TestFailMode
+    {
+        StopAfterFirstFailure,
+        AlwaysRunAllMoments
+    }
 }

ContinuousTests/ContinuousTestRunner.cs

@@ -1,6 +1,4 @@
-using DistTestCore;
+using Logging;
-using DistTestCore.Codex;
-using Logging;
 
 namespace ContinuousTests
 {
@@ -8,130 +6,33 @@ namespace ContinuousTests
     {
         private readonly ConfigLoader configLoader = new ConfigLoader();
         private readonly TestFactory testFactory = new TestFactory();
-        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly Configuration config;
+        private readonly StartupChecker startupChecker;
+
+        public ContinuousTestRunner(string[] args)
+        {
+            config = configLoader.Load(args);
+            startupChecker = new StartupChecker(config);
+        }
 
         public void Run()
         {
-            var config = //configLoader.Load();
+            startupChecker.Check();
-                new Configuration
-                {
-                    CodexUrls =new[] { "http://localhost:8080", "http://localhost:8081" },
-                    LogPath = "logs",
-                    KeepPassedTestLogs = false,
-                    SleepSecondsPerAllTests = 1,
-                    SleepSecondsPerSingleTest = 1,
-                };
-            StartupChecks(config);
 
-            while (true)
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
+            overviewLog.Log("Continuous tests starting...");
+            var allTests = testFactory.CreateTests();
+            var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
+
+            foreach (var t in testLoop)
             {
-                var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun");
+                overviewLog.Log("Launching test-loop for " + t.Name);
-                var allTestsRun = new AllTestsRun(config, log, testFactory);
+                t.Begin();
-
+                Thread.Sleep(TimeSpan.FromMinutes(5));
-                var result = ContinuousTestResult.Passed;
-                try
-                {
-                    result = allTestsRun.RunAll();
-                }
-                catch (Exception ex)
-                {
-                    log.Error($"Exception during test run: " + ex);
-                }
-
-                if (result == ContinuousTestResult.Failed)
-                {
-                    log.MarkAsFailed();
-                }
-                if (!config.KeepPassedTestLogs && result == ContinuousTestResult.Passed)
-                {
-                    log.DeleteFolder();
-                }
-
-                Thread.Sleep(config.SleepSecondsPerSingleTest * 1000);
             }
+
+            overviewLog.Log("All test-loops launched.");
+            while (true) Thread.Sleep((2 ^ 31) - 1);
         }
-
-        private void StartupChecks(Configuration config)
-        {
-            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
-            log.Log("Starting continuous test run...");
-            log.Log("Checking configuration...");
-            PreflightCheck(config);
-            log.Log("Contacting Codex nodes...");
-            CheckCodexNodes(log, config);
-            log.Log("All OK.");
-        }
-
-        private void PreflightCheck(Configuration config)
-        {
-            var tests = testFactory.CreateTests();
-            if (!tests.Any())
-            {
-                throw new Exception("Unable to find any tests.");
-            }
-
-            var errors = new List<string>();
-            foreach (var test in tests)
-            {
-                if (test.RequiredNumberOfNodes > config.CodexUrls.Length)
-                {
-                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Configuration only has {config.CodexUrls.Length}");
-                }
-            }
-
-            if (!Directory.Exists(config.LogPath))
-            {
-                Directory.CreateDirectory(config.LogPath);
-            }
-
-            if (errors.Any())
-            {
-                throw new Exception("Prerun check failed: " + string.Join(", ", errors));
-            }
-        }
-
-        private void CheckCodexNodes(BaseLog log, Configuration config)
-        {
-            var nodes = codexNodeFactory.Create(config.CodexUrls, log, new DefaultTimeSet());
-            var pass = true;
-            foreach (var n in nodes)
-            {
-                log.Log($"Checking '{n.Address.Host}'...");
-
-                if (EnsureOnline(n))
-                {
-                    log.Log("OK");
-                }
-                else
-                {
-                    log.Error($"No response from '{n.Address.Host}'.");
-                    pass = false;
-                }
-            }
-            if (!pass)
-            {
-                throw new Exception("Not all codex nodes responded.");
-            }
-        }
-
-        private bool EnsureOnline(CodexNode n)
-        {
-            try
-            {
-                var info = n.GetDebugInfo();
-                if (info == null || string.IsNullOrEmpty(info.id)) return false;
-            }
-            catch
-            {
-                return false;
-            }
-            return true;
-        }
-    }
-
-    public enum ContinuousTestResult
-    {
-        Passed,
-        Failed
     }
 }

ContinuousTests/ContinuousTests.csproj

@@ -12,7 +12,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
   </ItemGroup>
 

ContinuousTests/Program.cs

@@ -4,7 +4,9 @@ public class Program
 {
     public static void Main(string[] args)
     {
-        var runner = new ContinuousTestRunner();
+        Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Running...");
+        var runner = new ContinuousTestRunner(args);
         runner.Run();
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -2,55 +2,189 @@
 using DistTestCore;
 using Logging;
 using Utils;
+using KubernetesWorkflow;
+using NUnit.Framework.Internal;
+using System.Reflection;
 
 namespace ContinuousTests
 {
     public class SingleTestRun
     {
         private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly List<Exception> exceptions = new List<Exception>();
         private readonly Configuration config;
-        private readonly ContinuousTest test;
+        private readonly BaseLog overviewLog;
+        private readonly TestHandle handle;
         private readonly CodexNode[] nodes;
         private readonly FileManager fileManager;
+        private readonly FixtureLog fixtureLog;
+        private readonly string testName;
+        private readonly string dataFolder;
 
-        public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog)
+        public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle)
         {
             this.config = config;
-            this.test = test;
+            this.overviewLog = overviewLog;
+            this.handle = handle;
 
-            nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog);
+            testName = handle.Test.GetType().Name;
-            fileManager = new FileManager(testLog, new DistTestCore.Configuration());
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
 
-            test.Initialize(nodes, testLog, fileManager);
+            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            dataFolder = config.DataPath + "-" + Guid.NewGuid();
+            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
 
         public void Run()
         {
-            test.Run();
+            Task.Run(() =>
+            {
+                try
+                {
+                    RunTest();
+                    fileManager.DeleteAllTestFiles();
+                    Directory.Delete(dataFolder, true);
+                }
+                catch (Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
         }
 
-        public void TearDown()
+        private void RunTest()
         {
-            test.Initialize(null!, null!, null!);
+            try
-            fileManager.DeleteAllTestFiles();
+            {
+                RunTestMoments();
+
+                if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+            }
+            catch (Exception ex)
+            {
+                fixtureLog.Error("Test run failed with exception: " + ex);
+                fixtureLog.MarkAsFailed();
+            }
         }
 
-        private CodexNode[] CreateRandomNodes(int number, BaseLog testLog)
+        private void RunTestMoments()
         {
-            var urls = SelectRandomUrls(number);
+            var earliestMoment = handle.GetEarliestMoment();
-            testLog.Log("Selected nodes: " + string.Join(",", urls));
+
-            return codexNodeFactory.Create(urls, testLog, test.TimeSet);
+            var t = earliestMoment;
+            while (true)
+            {
+                RunMoment(t);
+
+                if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
+                {
+                    Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
+                    ThrowFailTest();
+                }
+
+                var nextMoment = handle.GetNextMoment(t);
+                if (nextMoment != null)
+                {
+                    Log($" > Next TestMoment in {nextMoment.Value} seconds...");
+                    t += nextMoment.Value;
+                    Thread.Sleep(nextMoment.Value * 1000);
+                }
+                else
+                {
+                    if (exceptions.Any())
+                    {
+                        ThrowFailTest();
+                    }
+                    OverviewLog(" > Test passed.");
+                    return;
+                }
+            }
         }
 
-        private string[] SelectRandomUrls(int number)
+        private void ThrowFailTest()
         {
-            var urls = config.CodexUrls.ToList();
+            var ex = UnpackException(exceptions.First());
-            var result = new string[number];
+            Log(ex.ToString());
+            OverviewLog(" > Test failed: " + ex.Message);
+            throw ex;
+        }
+
+        private Exception UnpackException(Exception exception)
+        {
+            if (exception is AggregateException a)
+            {
+                return UnpackException(a.InnerExceptions.First());
+            }
+            if (exception is TargetInvocationException t)
+            {
+                return UnpackException(t.InnerException!);
+            }
+
+            return exception;
+        }
+
+        private void RunMoment(int t)
+        {
+            using (var context = new TestExecutionContext.IsolatedContext())
+            {
+                try
+                {
+                    handle.InvokeMoment(t, InitializeTest);
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+            }
+
+            DecommissionTest();
+        }
+
+        private void InitializeTest(string name)
+        {
+            Log($" > Running TestMoment '{name}'");
+            handle.Test.Initialize(nodes, fixtureLog, fileManager, config);
+        }
+
+        private void DecommissionTest()
+        {
+            handle.Test.Initialize(null!, null!, null!, null!);
+        }
+
+        private void Log(string msg)
+        {
+            fixtureLog.Log(msg);
+        }
+
+        private void OverviewLog(string msg)
+        {
+            Log(msg);
+            overviewLog.Log(testName + ": " +  msg);
+        }
+
+        private CodexNode[] CreateRandomNodes(int number)
+        {
+            var containers = SelectRandomContainers(number);
+            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
+            return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
+        }
+
+        private RunningContainer[] SelectRandomContainers(int number)
+        {
+            var containers = config.CodexDeployment.CodexContainers.ToList();
+            var result = new RunningContainer[number];
             for (var i = 0; i < number; i++)
             {
-                result[i] = urls.PickOneRandom();
+                result[i] = containers.PickOneRandom();
             }
             return result;
         }
+
+        private DistTestCore.Configuration CreateFileManagerConfiguration()
+        {
+            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+                CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
+        }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -0,0 +1,102 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class StartupChecker
+    {
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly Configuration config;
+
+        public StartupChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Check()
+        {
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+            log.Log("Starting continuous test run...");
+            log.Log("Checking configuration...");
+            PreflightCheck(config);
+            log.Log("Contacting Codex nodes...");
+            CheckCodexNodes(log, config);
+            log.Log("All OK.");
+        }
+
+        private void PreflightCheck(Configuration config)
+        {
+            var tests = testFactory.CreateTests();
+            if (!tests.Any())
+            {
+                throw new Exception("Unable to find any tests.");
+            }
+            foreach (var test in tests)
+            {
+                var handle = new TestHandle(test);
+                handle.GetEarliestMoment();
+                handle.GetLastMoment();
+            }
+
+            var errors = new List<string>();
+            foreach (var test in tests)
+            {
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                {
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                }
+            }
+
+            if (!Directory.Exists(config.LogPath))
+            {
+                Directory.CreateDirectory(config.LogPath);
+            }
+
+            if (errors.Any())
+            {
+                throw new Exception("Prerun check failed: " + string.Join(", ", errors));
+            }
+        }
+
+        private void CheckCodexNodes(BaseLog log, Configuration config)
+        {
+            var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+            var pass = true;
+            foreach (var n in nodes)
+            {
+                log.Log($"Checking '{n.Address.Host}'...");
+
+                if (EnsureOnline(n))
+                {
+                    log.Log("OK");
+                }
+                else
+                {
+                    log.Error($"No response from '{n.Address.Host}'.");
+                    pass = false;
+                }
+            }
+            if (!pass)
+            {
+                throw new Exception("Not all codex nodes responded.");
+            }
+        }
+
+        private bool EnsureOnline(CodexNode n)
+        {
+            try
+            {
+                var info = n.GetDebugInfo();
+                if (info == null || string.IsNullOrEmpty(info.id)) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+    }
+}

ContinuousTests/TestHandle.cs

@@ -0,0 +1,86 @@
+using System.Reflection;
+
+namespace ContinuousTests
+{
+    public class TestHandle
+    {
+        private readonly List<MethodMoment> moments = new List<MethodMoment>();
+
+        public TestHandle(ContinuousTest test)
+        {
+            Test = test;
+
+            ReflectTestMoments();
+
+            var testName = test.GetType().Name;
+            if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
+            if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
+        }
+
+        public ContinuousTest Test { get; }
+
+        public int GetEarliestMoment()
+        {
+            return moments.Min(m => m.Moment);
+        }
+
+        public int GetLastMoment()
+        {
+            return moments.Max(m => m.Moment);
+        }
+
+        public int? GetNextMoment(int currentMoment)
+        {
+            var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
+            if (!remainingMoments.Any()) return null;
+            return remainingMoments.Min(m => m.Moment);
+        }
+
+        public void InvokeMoment(int currentMoment, Action<string> beforeInvoke)
+        {
+            var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
+            if (moment == null) return;
+
+            lock (MomentLock.Lock)
+            {
+                beforeInvoke(moment.Method.Name);
+                moment.Method.Invoke(Test, Array.Empty<object>());
+            }
+        }
+
+        private void ReflectTestMoments()
+        {
+            var methods = Test.GetType().GetMethods()
+                      .Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0)
+                      .ToArray();
+
+            foreach (var method in methods)
+            {
+                var moment = method.GetCustomAttribute<TestMomentAttribute>();
+                if (moment != null && moment.T >= 0)
+                {
+                    moments.Add(new MethodMoment(method, moment.T));
+                }
+            }
+        }
+    }
+
+    public class MethodMoment
+    {
+        public MethodMoment(MethodInfo method, int moment)
+        {
+            Method = method;
+            Moment = moment;
+
+            if (moment < 0) throw new Exception("Moment must be zero or greater.");
+        }
+
+        public MethodInfo Method { get; }
+        public int Moment { get; }
+    }
+
+    public static class MomentLock
+    {
+        public static readonly object Lock = new();
+    }
+}

ContinuousTests/TestLoop.cs

@@ -0,0 +1,52 @@
+using Logging;
+
+namespace ContinuousTests
+{
+    public class TestLoop
+    {
+        private readonly Configuration config;
+        private readonly BaseLog overviewLog;
+        private readonly Type testType;
+        private readonly TimeSpan runsEvery;
+
+        public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
+        {
+            this.config = config;
+            this.overviewLog = overviewLog;
+            this.testType = testType;
+            this.runsEvery = runsEvery;
+
+            Name = testType.Name;
+        }
+
+        public string Name { get; }
+
+        public void Begin()
+        {
+            Task.Run(() =>
+            {
+                try
+                {
+                    while (true)
+                    {
+                        StartTest();
+                        Thread.Sleep(runsEvery);
+                    }
+                }
+                catch(Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
+        }
+
+        private void StartTest()
+        {
+            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
+            var handle = new TestHandle(test);
+            var run = new SingleTestRun(config, overviewLog, handle);
+            run.Run();
+        }
+    }
+}

ContinuousTests/TestMomentAttribute.cs

@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestMomentAttribute : Attribute
+    {
+        public TestMomentAttribute(int t)
+        {
+            T = t;
+        }
+
+        public int T { get; }
+    }
+}

ContinuousTests/Tests/MarketplaceTest.cs

@@ -0,0 +1,150 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class MarketplaceTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromDays(4);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
+        public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
+
+        private readonly uint numberOfSlots = 3;
+        private readonly ByteSize fileSize = 10.MB();
+        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
+
+        private TestFile file = null!;
+        private ContentId? cid;
+        private string purchaseId = string.Empty;
+
+        [TestMoment(t: Zero)]
+        public void NodePostsStorageRequest()
+        {
+            var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
+            decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
+            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
+            Log.Log("expected total cost: " + expectedTotalCost);
+
+            file = FileManager.GenerateTestFile(fileSize);
+
+            var (workflowCreator, lifecycle) = CreateFacilities();
+            var flow = workflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
+                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
+                codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+
+                var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
+                var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
+
+                var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                interaction.MintTestTokens(new[] { account.Account }, expectedTotalCost, tokenAddress);
+
+                var container = rc.Containers[0];
+                var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork;
+                var codexAccess = new CodexAccess(lifecycle, container);
+                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
+
+                cid = UploadFile(codexAccess.Node, file);
+                Assert.That(cid, Is.Not.Null);
+
+                var balance = marketAccess.GetBalance();
+                Log.Log("Account: " + account.Account);
+                Log.Log("Balance: " + balance);
+
+                purchaseId = marketAccess.RequestStorage(
+                    contentId: cid!,
+                    pricePerSlotPerSecond: pricePerSlotPerSecond,
+                    requiredCollateral: 100.TestTokens(),
+                    minRequiredNumberOfNodes: numberOfSlots,
+                    proofProbability: 10,
+                    duration: contractDuration);
+
+                Log.Log($"PurchaseId: '{purchaseId}'");
+                Assert.That(!string.IsNullOrEmpty(purchaseId));
+            }
+            finally
+            {
+                flow.DeleteTestResources();
+            }
+        }
+
+        [TestMoment(t: DayThree)]
+        public void StoredDataIsAvailableAfterThreeDays()
+        {
+            var (workflowCreator, lifecycle) = CreateFacilities();
+            var flow = workflowCreator.CreateWorkflow();
+          
+            try
+            {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+                var container = rc.Containers[0];
+                var codexAccess = new CodexAccess(lifecycle, container);
+
+                var result = DownloadContent(codexAccess.Node, cid!);
+
+                file.AssertIsEqual(result);
+            }
+            finally
+            {
+                flow.DeleteTestResources();
+            }
+        }
+
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        {
+            var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: Configuration.LogPath,
+                codexLogLevel: CodexLogLevel.Debug,
+                runnerLocation: TestRunnerLocation.ExternalToCluster
+            );
+
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: MarketplaceTestNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: TimeSet.K8sOperationTimeout(),
+            retryDelay: TimeSet.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
+
+            return (workflowCreator, lifecycle);
+        }
+
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+    }
+}

ContinuousTests/Tests/PerformanceTests.cs

@@ -8,7 +8,8 @@ namespace ContinuousTests.Tests
     {
         public override int RequiredNumberOfNodes => 1;
 
-        public override void Run()
+        [TestMoment(t: Zero)]
+        public void UploadTest()
         {
             UploadTest(100, Nodes[0]);
         }
@@ -18,7 +19,8 @@ namespace ContinuousTests.Tests
     {
         public override int RequiredNumberOfNodes => 1;
 
-        public override void Run()
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
         {
             DownloadTest(100, Nodes[0], Nodes[0]);
         }
@@ -28,7 +30,8 @@ namespace ContinuousTests.Tests
     {
         public override int RequiredNumberOfNodes => 2;
 
-        public override void Run()
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
         {
             DownloadTest(100, Nodes[0], Nodes[1]);
         }
@@ -36,6 +39,9 @@ namespace ContinuousTests.Tests
 
     public abstract class PerformanceTest : ContinuousTest
     {
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
         public void UploadTest(int megabytes, CodexNode uploadNode)
         {
             var file = FileManager.GenerateTestFile(megabytes.MB());

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,14 +6,24 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
-        public override void Run()
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
         {
-            var file = FileManager.GenerateTestFile(10.MB());
+            file = FileManager.GenerateTestFile(10.MB());
 
-            var cid = UploadFile(Nodes[0], file);
+            cid = UploadFile(Nodes[0], file);
             Assert.That(cid, Is.Not.Null);
+        }
 
+        [TestMoment(t: MinuteFive)]
+        public void DownloadTestFile()
+        {
             var dl = DownloadContent(Nodes[1], cid!);
 
             file.AssertIsEqual(dl);

DistTestCore/Codex/CodexNode.cs

@@ -65,6 +65,11 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
+        }
+
         public string ConnectToPeer(string peerId, string peerMultiAddress)
         {
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
@@ -170,4 +175,10 @@ namespace DistTestCore.Codex
         public uint? nodes { get; set; }
         public uint? tolerance { get; set; }
     }
+
+    public class CodexStoragePurchase
+    {
+        public string state { get; set; } = string.Empty;
+        public string error { get; set; } = string.Empty;
+    }
 }

DistTestCore/DistTest.cs

@@ -250,7 +250,7 @@ namespace DistTestCore
         {
             OnEachCodexNode(lifecycle, node =>
             {
-                lifecycle.DownloadLog(node);
+                lifecycle.DownloadLog(node.CodexAccess.Container);
             });
         }
 

DistTestCore/Logs/DownloadedLog.cs

@@ -3,17 +3,17 @@ using NUnit.Framework;
 
 namespace DistTestCore.Logs
 {
-    public interface ICodexNodeLog
+    public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
     }
 
-    public class CodexNodeLog : ICodexNodeLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
-        private readonly OnlineCodexNode owner;
+        private readonly string owner;
 
-        public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
+        public DownloadedLog(LogFile logFile, string owner)
         {
             this.logFile = logFile;
             this.owner = owner;
@@ -31,7 +31,7 @@ namespace DistTestCore.Logs
                 line = streamReader.ReadLine();
             }
 
-            Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
+            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
     }
 }

DistTestCore/Logs/LogDownloadHandler.cs

@@ -5,21 +5,21 @@ namespace DistTestCore.Logs
 {
     public class LogDownloadHandler : LogHandler, ILogHandler
     {
-        private readonly OnlineCodexNode node;
+        private readonly RunningContainer container;
         private readonly LogFile log;
 
-        public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
+        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
         {
-            this.node = node;
+            this.container = container;
             this.log = log;
 
             log.Write($"{description} -->> {log.FullFilename}");
             log.WriteRaw(description);
         }
 
-        public CodexNodeLog CreateCodexNodeLog()
+        public DownloadedLog DownloadLog()
         {
-            return new CodexNodeLog(log, node);
+            return new DownloadedLog(log, container.Name);
         }
 
         protected override void ProcessLine(string line)

DistTestCore/Marketplace/GethStartResult.cs

@@ -1,4 +1,6 @@
-namespace DistTestCore.Marketplace
+using Newtonsoft.Json;
+
+namespace DistTestCore.Marketplace
 {
     public class GethStartResult
     {
@@ -9,6 +11,7 @@
             CompanionNode = companionNode;
         }
 
+        [JsonIgnore]
         public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
         public MarketplaceNetwork MarketplaceNetwork { get; }
         public GethCompanionNodeInfo CompanionNode { get; }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToHexBigInt(duration.TotalSeconds),
                 proofProbability = ToHexBigInt(proofProbability),
-                reward = ToHexBigInt(pricePerBytePerSecond),
+                reward = ToHexBigInt(pricePerSlotPerSecond),
                 collateral = ToHexBigInt(requiredCollateral),
                 expiry = null,
                 nodes = minRequiredNumberOfNodes,
@@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
             };
 
             Log($"Requesting storage for: {contentId.Id}... (" +
-                $"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
+                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +

DistTestCore/OnlineCodexNode.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
+        IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         ICodexSetup BringOffline();
@@ -107,9 +107,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public ICodexNodeLog DownloadLog()
+        public IDownloadedLog DownloadLog()
         {
-            return lifecycle.DownloadLog(this);
+            return lifecycle.DownloadLog(CodexAccess.Container);
         }
 
         public ICodexSetup BringOffline()

DistTestCore/TestLifecycle.cs

@@ -41,16 +41,16 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public ICodexNodeLog DownloadLog(OnlineCodexNode node)
+        public IDownloadedLog DownloadLog(RunningContainer container)
         {
             var subFile = Log.CreateSubfile();
-            var description = node.GetName();
+            var description = container.Name;
-            var handler = new LogDownloadHandler(node, description, subFile);
+            var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
+            CodexStarter.DownloadLog(container, handler);
 
-            return new CodexNodeLog(subFile, node);
+            return new DownloadedLog(subFile, description);
         }
 
         public string GetTestDuration()

KubernetesWorkflow/K8sController.cs

@@ -39,7 +39,7 @@ namespace KubernetesWorkflow
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
             var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
         }
 
         public void Stop(RunningPod pod)
@@ -436,9 +436,9 @@ namespace KubernetesWorkflow
 
         #region Service management
 
-        private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
+        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
         {
-            var result = new Dictionary<ContainerRecipe, Port[]>();
+            var result = new List<ContainerRecipePortMapEntry>();
 
             var ports = CreateServicePorts(containerRecipes);
 
@@ -468,7 +468,7 @@ namespace KubernetesWorkflow
             return (serviceSpec.Metadata.Name, result);
         }
 
-        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
         {
             // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@@ -485,7 +485,8 @@ namespace KubernetesWorkflow
                         // These service ports belongs to this recipe.
                         var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
-                        result.Add(r, ports);
+
+                        result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
                     }
                 }
             }

KubernetesWorkflow/RunningContainers.cs

@@ -23,12 +23,12 @@ namespace KubernetesWorkflow
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, Address clusterExternalAddress, Address clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
-            Name = GetContainerName(recipe, startupConfig);
+            Name = name;
             ClusterExternalAddress = clusterExternalAddress;
             ClusterInternalAddress = clusterInternalAddress;
         }
@@ -39,17 +39,5 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
-
-        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
-        {
-            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
-            {
-                return $"<{startupConfig.NameOverride}{recipe.Number}>";
-            }
-            else
-            {
-                return $"<{recipe.Name}>";
-            }
-        }
     }
 }

KubernetesWorkflow/RunningPod.cs

@@ -2,35 +2,50 @@
 {
     public class RunningPod
     {
-        private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
-
-        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
             PodInfo = podInfo;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
-            this.servicePortMap = servicePortMap;
+            PortMapEntries = portMapEntries;
         }
 
         public K8sCluster Cluster { get; }
         public PodInfo PodInfo { get; }
+        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
+            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
-            return servicePortMap[containerRecipe];
+            {
+                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
+            }
+
+            return Array.Empty<Port>();
         }
     }
 
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
+        {
+            ContainerNumber = containerNumber;
+            Ports = ports;
+        }
+
+        public int ContainerNumber { get; }
+        public Port[] Ports { get; }
+    }
+
     public class PodInfo
     {
-        public PodInfo(string podName, string podIp, string k8sNodeName)
+        public PodInfo(string name, string ip, string k8sNodeName)
         {
-            Name = podName;
+            Name = name;
-            Ip = podIp;
+            Ip = ip;
             K8SNodeName = k8sNodeName;
         }
 

KubernetesWorkflow/StartupWorkflow.cs

@@ -81,13 +81,28 @@ namespace KubernetesWorkflow
                 var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
                 log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, startupConfig,
+                var name = GetContainerName(r, startupConfig);
+
+                return new RunningContainer(runningPod, r, servicePorts, name,
                     GetContainerExternalAddress(runningPod, servicePorts),
                     GetContainerInternalAddress(r));
 
             }).ToArray();
         }
 
+        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
+        {
+            if (startupConfig == null) return "";
+            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
+            {
+                return $"<{startupConfig.NameOverride}{recipe.Number}>";
+            }
+            else
+            {
+                return $"<{recipe.Name}>";
+            }
+        }
+
         private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
         {
             return new Address(

Logging/NullLog.cs

@@ -1,6 +1,4 @@
-using Logging;
+namespace Logging
-
-namespace CodexNetDeployer
 {
     public class NullLog : TestLog
     {
@@ -15,12 +13,10 @@ namespace CodexNetDeployer
 
         public override void Log(string message)
         {
-            //Console.WriteLine(message);
         }
 
         public override void Debug(string message = "", int skipFrames = 0)
         {
-            //Console.WriteLine(message);
         }
 
         public override void Error(string message)

Tests/BasicTests/ExampleTests.cs

@@ -64,7 +64,7 @@ namespace Tests.BasicTests
             
             var contentId = buyer.UploadFile(testFile);
             buyer.Marketplace.RequestStorage(contentId,
-                pricePerBytePerSecond: 2.TestTokens(),
+                pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,

cs-codex-dist-testing.sln

@@ -19,7 +19,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nether
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -63,6 +65,10 @@ Global
 		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE