Merge branch 'feature/continuous-testing'

2023-06-25 10:56:45 +02:00 · 2023-06-25 10:56:45 +02:00 · 4e2e4830a1
parent 62d646836d ae3b4df92c
commit 4e2e4830a1
46 changed files with 1692 additions and 234 deletions
--- a/CodexNetDeployer/ArgOrVar.cs
+++ b/CodexNetDeployer/ArgOrVar.cs
@ -0,0 +1,93 @@
+namespace CodexNetDeployer
+{
+    public class ArgOrVar
+    {
+        public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex.");
+        public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth.");
+        public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer.");
+        public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file.");
+        public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment.");
+        public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created.");
+        public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating.");
+        public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node.");
+        public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]");
+
+        private readonly string[] args;
+
+        public ArgOrVar(string[] args)
+        {
+            this.args = args;
+        }
+
+        public string Get(ArgVar key, string defaultValue = "")
+        {
+            var argKey = $"--{key.Arg}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+
+            var env = Environment.GetEnvironmentVariable(key.Var);
+            if (env != null)
+            {
+                return env;
+            }
+
+            return defaultValue;
+        }
+        
+        public int? GetInt(ArgVar key)
+        {
+            var str = Get(key);
+            if (string.IsNullOrEmpty(str)) return null;
+            if (int.TryParse(str, out int result))
+            {
+                return result;
+            }
+            return null;
+        }
+
+        public void PrintHelp()
+        {
+            var nl = Environment.NewLine;
+            Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+                "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
+                "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+            Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+
+            Console.Write("\t[ CLI argument ] or [ Environment variable ]");
+            Console.CursorLeft = 70;
+            Console.Write("(Description)" + nl);
+            var fields = GetType().GetFields();
+            foreach (var field in fields)
+            {
+                var value = (ArgVar)field.GetValue(null)!;
+                value.PrintHelp();
+            }
+        }
+    }
+
+    public class ArgVar
+    {
+        public ArgVar(string arg, string var, string description)
+        {
+            Arg = arg;
+            Var = var;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string Var { get; }
+        public string Description { get; }
+
+        public void PrintHelp()
+        {
+            Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]");
+            Console.CursorLeft = 70;
+            Console.Write(Description + Environment.NewLine);
+        }
+    }
+}
--- a/CodexNetDeployer/CodexNetDeployer.csproj
+++ b/CodexNetDeployer/CodexNetDeployer.csproj
@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>
--- a/CodexNetDeployer/CodexNodeStarter.cs
+++ b/CodexNetDeployer/CodexNodeStarter.cs
@ -0,0 +1,75 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class CodexNodeStarter
+    {
+        private readonly Configuration config;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly TestLifecycle lifecycle;
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+        private readonly GethStartResult gethResult;
+        private string bootstrapSpr = "";
+        private int validatorsLeft;
+
+        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators)
+        {
+            this.config = config;
+            this.workflowCreator = workflowCreator;
+            this.lifecycle = lifecycle;
+            this.log = log;
+            this.timeSet = timeSet;
+            this.gethResult = gethResult;
+            this.validatorsLeft = numberOfValidators;
+        }
+
+        public RunningContainer? Start(int i)
+        {
+            Console.Write($" - {i} = ");
+            var workflow = workflowCreator.CreateWorkflow();
+            var workflowStartup = new StartupConfig();
+            workflowStartup.Add(gethResult);
+            workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
+
+            var container = containers.Containers.First();
+            var address = lifecycle.Configuration.GetAddress(container);
+            var codexNode = new CodexNode(log, timeSet, address);
+            var debugInfo = codexNode.GetDebugInfo();
+
+            if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+            {
+                var pod = container.Pod.PodInfo;
+                Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine);
+
+                if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+                validatorsLeft--;
+                return container;
+            }
+            else
+            {
+                Console.Write("Unknown failure." + Environment.NewLine);
+                return null;
+            }
+        }
+
+        private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
+        {
+            var codexStart = new CodexStartupConfig(config.CodexLogLevel);
+
+            if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
+            codexStart.StorageQuota = config.StorageQuota!.Value.MB();
+            var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
+            marketplaceConfig.AccountIndexOverride = i;
+            codexStart.MarketplaceConfig = marketplaceConfig;
+
+            return codexStart;
+        }
+    }
+}
--- a/CodexNetDeployer/Configuration.cs
+++ b/CodexNetDeployer/Configuration.cs
@ -0,0 +1,101 @@
+using DistTestCore;
+using DistTestCore.Codex;
+
+namespace CodexNetDeployer
+{
+    public class Configuration
+    {
+        public Configuration(
+            string codexImage,
+            string gethImage,
+            string contractsImage,
+            string kubeConfigFile,
+            string kubeNamespace,
+            int? numberOfCodexNodes,
+            int? numberOfValidators,
+            int? storageQuota,
+            CodexLogLevel codexLogLevel,
+            TestRunnerLocation runnerLocation)
+        {
+            CodexImage = codexImage;
+            GethImage = gethImage;
+            ContractsImage = contractsImage;
+            KubeConfigFile = kubeConfigFile;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuota = storageQuota;
+            CodexLogLevel = codexLogLevel;
+            RunnerLocation = runnerLocation;
+        }
+
+        public string CodexImage { get; }
+        public string GethImage { get; }
+        public string ContractsImage { get; }
+        public string KubeConfigFile { get; }
+        public string KubeNamespace { get; }
+        public int? NumberOfCodexNodes { get; }
+        public int? NumberOfValidators { get; }
+        public int? StorageQuota { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+        public TestRunnerLocation RunnerLocation { get; }
+
+        public void PrintConfig()
+        {
+            ForEachProperty(onString: Print, onInt: Print);
+        }
+
+        public List<string> Validate()
+        {
+            var errors = new List<string>();
+
+            ForEachProperty(
+                onString: (n, v) => StringIsSet(n, v, errors),
+                onInt: (n, v) => IntIsOverZero(n, v, errors));
+
+            if (NumberOfValidators > NumberOfCodexNodes)
+            {
+                errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
+            }
+
+            return errors;
+        }
+
+        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
+        {
+            var properties = GetType().GetProperties();
+            foreach (var p in properties)
+            {
+                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
+            }
+        }
+
+        private static void IntIsOverZero(string variable, int? value, List<string> errors)
+        {
+            if (value == null || value.Value < 1)
+            {
+                errors.Add($"{variable} is must be set and must be greater than 0.");
+            }
+        }
+
+        private static void StringIsSet(string variable, string value, List<string> errors)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                errors.Add($"{variable} is must be set.");
+            }
+        }
+
+        private static void Print(string variable, string value)
+        {
+            Console.WriteLine($"\t{variable}: '{value}'");
+        }
+
+        private static void Print(string variable, int? value)
+        {
+            if (value != null) Print(variable, value.ToString()!);
+            else Print(variable, "<NONE>");
+        }
+    }
+}
--- a/CodexNetDeployer/Deployer.cs
+++ b/CodexNetDeployer/Deployer.cs
@ -0,0 +1,93 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace CodexNetDeployer
+{
+    public class Deployer
+    {
+        private readonly Configuration config;
+        private readonly NullLog log;
+        private readonly DefaultTimeSet timeset;
+
+        public Deployer(Configuration config)
+        {
+            this.config = config;
+            log = new NullLog();
+            timeset = new DefaultTimeSet();
+        }
+
+        public CodexDeployment Deploy()
+        {
+            Log("Initializing...");
+            var (workflowCreator, lifecycle) = CreateFacilities();
+
+            Log("Preparing configuration...");
+            // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
+            var setup = new CodexSetup(999, config.CodexLogLevel);
+            setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+
+            Log("Creating Geth instance and deploying contracts...");
+            var gethStarter = new GethStarter(lifecycle, workflowCreator);
+            var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
+
+            Log("Geth started. Codex contracts deployed.");
+            Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
+
+            Log("Starting Codex nodes...");
+
+            // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
+            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value);
+            var codexContainers = new List<RunningContainer>();
+            for (var i = 0; i < config.NumberOfCodexNodes; i++)
+            {
+                var container = codexStarter.Start(i);
+                if (container != null) codexContainers.Add(container);
+            }
+
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata());
+        }
+
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        {
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: config.KubeConfigFile,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: "notUsed",
+                codexLogLevel: config.CodexLogLevel,
+                runnerLocation: config.RunnerLocation
+            );
+
+            var kubeConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: config.KubeNamespace,
+                kubeConfigFile: config.KubeConfigFile,
+                operationTimeout: timeset.K8sOperationTimeout(),
+                retryDelay: timeset.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
+
+            return (workflowCreator, lifecycle);
+        }
+
+        private DeploymentMetadata CreateMetadata()
+        {
+            return new DeploymentMetadata(
+                codexImage: config.CodexImage,
+                gethImage: config.GethImage,
+                contractsImage: config.ContractsImage,
+                kubeNamespace: config.KubeNamespace,
+                numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
+                numberOfValidators: config.NumberOfValidators!.Value,
+                storageQuotaMB: config.StorageQuota!.Value,
+                codexLogLevel: config.CodexLogLevel);
+        }
+
+        private void Log(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+    }
+}
--- a/CodexNetDeployer/NullLog.cs
+++ b/CodexNetDeployer/NullLog.cs
@ -0,0 +1,43 @@
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class NullLog : TestLog
+    {
+        public NullLog() : base("NULL", false, "NULL")
+        {
+        }
+
+        protected override LogFile CreateLogFile()
+        {
+            return null!;
+        }
+
+        public override void Log(string message)
+        {
+            //Console.WriteLine(message);
+        }
+
+        public override void Debug(string message = "", int skipFrames = 0)
+        {
+            //Console.WriteLine(message);
+        }
+
+        public override void Error(string message)
+        {
+            Console.WriteLine("Error: " + message);
+        }
+
+        public override void MarkAsFailed()
+        {
+        }
+
+        public override void AddStringReplace(string from, string to)
+        {
+        }
+
+        public override void Delete()
+        {
+        }
+    }
+}
--- a/CodexNetDeployer/Program.cs
+++ b/CodexNetDeployer/Program.cs
@ -0,0 +1,66 @@
+using CodexNetDeployer;
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using Newtonsoft.Json;
+using Utils;
+using Configuration = CodexNetDeployer.Configuration;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDeployer" + nl + nl);
+
+        var argOrVar = new ArgOrVar(args);
+
+        if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+        {
+            argOrVar.PrintHelp();
+            return;
+        }
+
+        var location = TestRunnerLocation.InternalToCluster;
+        if (args.Any(a => a == "--external"))
+        {
+            location = TestRunnerLocation.ExternalToCluster;
+        }
+
+        var config = new Configuration(
+            codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage),
+            gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage),
+            contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage),
+            kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile),
+            kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace),
+            numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes),
+            numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes),
+            storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota),
+            codexLogLevel: ParseEnum.Parse<CodexLogLevel>(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))),
+            runnerLocation: location
+        );
+
+        Console.WriteLine("Using:");
+        config.PrintConfig();
+        Console.WriteLine(nl);
+
+        var errors = config.Validate();
+        if (errors.Any())
+        {
+            Console.WriteLine($"Configuration errors: ({errors.Count})");
+            foreach ( var error in errors ) Console.WriteLine("\t" + error);
+            Console.WriteLine(nl);
+            argOrVar.PrintHelp();
+            return;
+        }
+
+        var deployer = new Deployer(config);
+        var deployment = deployer.Deploy();
+
+        Console.WriteLine("Writing codex-deployment.json...");
+
+        File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented));
+
+        Console.WriteLine("Done!");
+    }
+}
--- a/ContinuousTests/CodexNodeFactory.cs
+++ b/ContinuousTests/CodexNodeFactory.cs
@ -0,0 +1,19 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class CodexNodeFactory
+    {
+        public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+        {
+            return containers.Select(container =>
+            {
+                var address = container.ClusterExternalAddress;
+                return new CodexNode(log, timeSet, address);
+            }).ToArray();
+        }
+    }
+}
--- a/ContinuousTests/Configuration.cs
+++ b/ContinuousTests/Configuration.cs
@ -0,0 +1,98 @@
+using DistTestCore.Codex;
+using Newtonsoft.Json;
+
+namespace ContinuousTests
+{
+    public class Configuration
+    {
+        public string LogPath { get; set; } = string.Empty;
+        public string DataPath { get; set; } = string.Empty;
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+        public bool KeepPassedTestLogs { get; set; }
+    }
+
+    public class ConfigLoader
+    {
+        private const string filename = "config.json";
+
+        public Configuration Load()
+        {
+            var config = Read();
+
+            Validate(config);
+            return config;
+        }
+
+        private Configuration Read()
+        {
+            if (File.Exists(filename))
+            {
+                var lines = File.ReadAllText(filename);
+                try
+                {
+                    var result = JsonConvert.DeserializeObject<Configuration>(lines);
+                    if (result != null) return result;
+                }
+                catch { }
+            }
+
+            var logPath = Environment.GetEnvironmentVariable("LOGPATH");
+            var dataPath = Environment.GetEnvironmentVariable("DATAPATH");
+            var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT");
+            var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS");
+
+            if (!string.IsNullOrEmpty(logPath) &&
+                !string.IsNullOrEmpty(dataPath) &&
+                !string.IsNullOrEmpty(codexDeploymentJson))
+            {
+                try 
+                {
+                    return new Configuration
+                    { 
+                        LogPath = logPath,
+                        DataPath = dataPath,
+                        CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson),
+                        KeepPassedTestLogs = keep == "1"
+                    };
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine("Exception: " + ex);
+                }
+            }
+
+            var nl = Environment.NewLine;
+            throw new Exception($"Unable to load configuration from '{filename}', and " +
+                "unable to load configuration from environment variables. " + nl +
+                "'LOGPATH' = Path where log files will be saved." + nl +
+                "'DATAPATH' = Path where temporary data files will be saved." + nl +
+                "'CODEXDEPLOYMENT' = Path to codex-deployment JSON file." + nl +
+                nl);
+        }
+
+        private void Validate(Configuration configuration)
+        {
+            if (string.IsNullOrEmpty(configuration.LogPath))
+            {
+                throw new Exception($"Invalid LogPath set: '{configuration.LogPath}'");
+            }
+
+            if (string.IsNullOrEmpty(configuration.DataPath))
+            {
+                throw new Exception($"Invalid DataPath set: '{configuration.DataPath}'");
+            }
+
+            if (configuration.CodexDeployment == null || !configuration.CodexDeployment.CodexContainers.Any())
+            {
+                throw new Exception("No Codex deployment found.");
+            }
+        }
+
+        private CodexDeployment ParseCodexDeploymentJson(string filename)
+        {
+            var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+            if (d == null) throw new Exception("Unable to parse " + filename);
+            return d;
+        }
+    }
+}
--- a/ContinuousTests/ContinuousTest.cs
+++ b/ContinuousTests/ContinuousTest.cs
@ -0,0 +1,95 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+
+namespace ContinuousTests
+{
+    public abstract class ContinuousTestLongTimeouts : ContinuousTest
+    {
+        public override ITimeSet TimeSet => new LongTimeSet();
+    }
+
+    public abstract class ContinuousTest
+    {
+        protected const int Zero = 0;
+        protected const int MinuteOne = 60;
+        protected const int MinuteFive = MinuteOne * 5;
+        protected const int HourOne = MinuteOne * 60;
+        protected const int DayOne = HourOne * 24;
+        protected const int DayThree = DayOne * 3;
+
+        private const string UploadFailedMessage = "Unable to store block";
+
+        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager)
+        {
+            Nodes = nodes;
+            Log = log;
+            FileManager = fileManager;
+        }
+
+        public CodexNode[] Nodes { get; private set; } = null!;
+        public BaseLog Log { get; private set; } = null!;
+        public IFileManager FileManager { get; private set; } = null!;
+        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+
+        public abstract int RequiredNumberOfNodes { get; }
+        public abstract TimeSpan RunTestEvery { get; }
+        public abstract TestFailMode TestFailMode { get; }
+
+        public string Name
+        {
+            get
+            {
+                return GetType().Name;
+            }
+        }
+
+        public ContentId? UploadFile(CodexNode node, TestFile file)
+        {
+            using var fileStream = File.OpenRead(file.Filename);
+
+            var logMessage = $"Uploading file {file.Describe()}...";
+            var response = Stopwatch.Measure(Log, logMessage, () =>
+            {
+                return node.UploadFile(fileStream);
+            });
+
+            if (response.StartsWith(UploadFailedMessage))
+            {
+                return null;
+            }
+            Log.Log($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
+        {
+            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            var file = FileManager.CreateEmptyTestFile(fileLabel);
+            Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
+            Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+            return file;
+        }
+
+        private void DownloadToFile(CodexNode node, string contentId, TestFile file)
+        {
+            using var fileStream = File.OpenWrite(file.Filename);
+            try
+            {
+                using var downloadStream = node.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log.Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
+        }
+    }
+
+    public enum TestFailMode
+    {
+        StopAfterFirstFailure,
+        AlwaysRunAllMoments
+    }
+}
--- a/ContinuousTests/ContinuousTestRunner.cs
+++ b/ContinuousTests/ContinuousTestRunner.cs
@ -0,0 +1,31 @@
+namespace ContinuousTests
+{
+    public class ContinuousTestRunner
+    {
+        private readonly ConfigLoader configLoader = new ConfigLoader();
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly Configuration config;
+        private readonly StartupChecker startupChecker;
+
+        public ContinuousTestRunner()
+        {
+            config = configLoader.Load();
+            startupChecker = new StartupChecker(config);
+        }
+
+        public void Run()
+        {
+            startupChecker.Check();
+
+            var allTests = testFactory.CreateTests();
+            var testStarters = allTests.Select(t => new TestStarter(config, t.GetType(), t.RunTestEvery)).ToArray();
+
+            foreach (var t in testStarters)
+            {
+                t.Begin();
+            }
+          
+            while (true) Thread.Sleep((2 ^ 31) - 1);
+        }
+    }
+}
--- a/ContinuousTests/ContinuousTests.csproj
+++ b/ContinuousTests/ContinuousTests.csproj
@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>
--- a/ContinuousTests/Program.cs
+++ b/ContinuousTests/Program.cs
@ -0,0 +1,12 @@
+using ContinuousTests;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Running...");
+        var runner = new ContinuousTestRunner();
+        runner.Run();
+    }
+}
--- a/ContinuousTests/SingleTestRun.cs
+++ b/ContinuousTests/SingleTestRun.cs
@ -0,0 +1,143 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+using Utils;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+    public class SingleTestRun
+    {
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly List<Exception> exceptions = new List<Exception>();
+        private readonly Configuration config;
+        private readonly TestHandle handle;
+        private readonly CodexNode[] nodes;
+        private readonly FileManager fileManager;
+        private readonly FixtureLog fixtureLog;
+        private readonly string dataFolder;
+
+        public SingleTestRun(Configuration config, TestHandle handle)
+        {
+            this.config = config;
+            this.handle = handle;
+
+            var testName = handle.Test.GetType().Name;
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
+
+            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            dataFolder = config.DataPath + "-" + Guid.NewGuid();
+            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
+        }
+
+        public void Run()
+        {
+            Task.Run(() =>
+            {
+                try
+                {
+                    RunTest();
+
+                    if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+                }
+                catch (Exception ex)
+                {
+                    fixtureLog.Error("Test run failed with exception: " + ex);
+                    fixtureLog.MarkAsFailed();
+                }
+                fileManager.DeleteAllTestFiles();
+                Directory.Delete(dataFolder, true);
+            });
+        }
+
+        private void RunTest()
+        {
+            var earliestMoment = handle.GetEarliestMoment();
+
+            var t = earliestMoment;
+            while (true)
+            {
+                RunMoment(t);
+
+                if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
+                {
+                    Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
+                    throw exceptions.Single();
+                }
+
+                var nextMoment = handle.GetNextMoment(t);
+                if (nextMoment != null)
+                {
+                    Log($" > Next TestMoment in {nextMoment.Value} seconds...");
+                    t += nextMoment.Value;
+                    Thread.Sleep(nextMoment.Value * 1000);
+                }
+                else
+                {
+                    if (exceptions.Any())
+                    {
+                        Log(" > Completed last test moment. Test failed.");
+                        throw exceptions.First();
+                    }
+                    Log(" > Completed last test moment. Test passed.");
+                    return;
+                }
+            }
+        }
+
+        private void RunMoment(int t)
+        {
+            try
+            {
+                handle.InvokeMoment(t, InitializeTest);
+            }
+            catch (Exception ex)
+            {
+                Log($" > TestMoment yielded exception: " + ex);
+                exceptions.Add(ex);
+            }
+
+            DecommissionTest();
+        }
+
+        private void InitializeTest(string name)
+        {
+            Log($" > Running TestMoment '{name}'");
+            handle.Test.Initialize(nodes, fixtureLog, fileManager);
+        }
+
+        private void DecommissionTest()
+        {
+            handle.Test.Initialize(null!, null!, null!);
+        }
+
+        private void Log(string msg)
+        {
+            fixtureLog.Log(msg);
+        }
+
+        private CodexNode[] CreateRandomNodes(int number)
+        {
+            var containers = SelectRandomContainers(number);
+            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
+            return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
+        }
+
+        private RunningContainer[] SelectRandomContainers(int number)
+        {
+            var containers = config.CodexDeployment.CodexContainers.ToList();
+            var result = new RunningContainer[number];
+            for (var i = 0; i < number; i++)
+            {
+                result[i] = containers.PickOneRandom();
+            }
+            return result;
+        }
+
+        private DistTestCore.Configuration CreateFileManagerConfiguration()
+        {
+            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+                CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
+        }
+    }
+}
--- a/ContinuousTests/StartupChecker.cs
+++ b/ContinuousTests/StartupChecker.cs
@ -0,0 +1,102 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class StartupChecker
+    {
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly Configuration config;
+
+        public StartupChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Check()
+        {
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+            log.Log("Starting continuous test run...");
+            log.Log("Checking configuration...");
+            PreflightCheck(config);
+            log.Log("Contacting Codex nodes...");
+            CheckCodexNodes(log, config);
+            log.Log("All OK.");
+        }
+
+        private void PreflightCheck(Configuration config)
+        {
+            var tests = testFactory.CreateTests();
+            if (!tests.Any())
+            {
+                throw new Exception("Unable to find any tests.");
+            }
+            foreach (var test in tests)
+            {
+                var handle = new TestHandle(test);
+                handle.GetEarliestMoment();
+                handle.GetLastMoment();
+            }
+
+            var errors = new List<string>();
+            foreach (var test in tests)
+            {
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                {
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                }
+            }
+
+            if (!Directory.Exists(config.LogPath))
+            {
+                Directory.CreateDirectory(config.LogPath);
+            }
+
+            if (errors.Any())
+            {
+                throw new Exception("Prerun check failed: " + string.Join(", ", errors));
+            }
+        }
+
+        private void CheckCodexNodes(BaseLog log, Configuration config)
+        {
+            var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+            var pass = true;
+            foreach (var n in nodes)
+            {
+                log.Log($"Checking '{n.Address.Host}'...");
+
+                if (EnsureOnline(n))
+                {
+                    log.Log("OK");
+                }
+                else
+                {
+                    log.Error($"No response from '{n.Address.Host}'.");
+                    pass = false;
+                }
+            }
+            if (!pass)
+            {
+                throw new Exception("Not all codex nodes responded.");
+            }
+        }
+
+        private bool EnsureOnline(CodexNode n)
+        {
+            try
+            {
+                var info = n.GetDebugInfo();
+                if (info == null || string.IsNullOrEmpty(info.id)) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+    }
+}
--- a/ContinuousTests/TestFactory.cs
+++ b/ContinuousTests/TestFactory.cs
@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestFactory
+    {
+        public ContinuousTest[] CreateTests()
+        {
+            var types = GetType().Assembly.GetTypes();
+            var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract);
+            return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray();
+        }
+    }
+}
--- a/ContinuousTests/TestHandle.cs
+++ b/ContinuousTests/TestHandle.cs
@ -0,0 +1,86 @@
+using System.Reflection;
+
+namespace ContinuousTests
+{
+    public class TestHandle
+    {
+        private readonly List<MethodMoment> moments = new List<MethodMoment>();
+
+        public TestHandle(ContinuousTest test)
+        {
+            Test = test;
+
+            ReflectTestMoments();
+
+            var testName = test.GetType().Name;
+            if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
+            if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
+        }
+
+        public ContinuousTest Test { get; }
+
+        public int GetEarliestMoment()
+        {
+            return moments.Min(m => m.Moment);
+        }
+
+        public int GetLastMoment()
+        {
+            return moments.Max(m => m.Moment);
+        }
+
+        public int? GetNextMoment(int currentMoment)
+        {
+            var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
+            if (!remainingMoments.Any()) return null;
+            return remainingMoments.Min(m => m.Moment);
+        }
+
+        public void InvokeMoment(int currentMoment, Action<string> beforeInvoke)
+        {
+            var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
+            if (moment == null) return;
+
+            lock (MomentLock.Lock)
+            {
+                beforeInvoke(moment.Method.Name);
+                moment.Method.Invoke(Test, Array.Empty<object>());
+            }
+        }
+
+        private void ReflectTestMoments()
+        {
+            var methods = Test.GetType().GetMethods()
+                      .Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0)
+                      .ToArray();
+
+            foreach (var method in methods)
+            {
+                var moment = method.GetCustomAttribute<TestMomentAttribute>();
+                if (moment != null && moment.T >= 0)
+                {
+                    moments.Add(new MethodMoment(method, moment.T));
+                }
+            }
+        }
+    }
+
+    public class MethodMoment
+    {
+        public MethodMoment(MethodInfo method, int moment)
+        {
+            Method = method;
+            Moment = moment;
+
+            if (moment < 0) throw new Exception("Moment must be zero or greater.");
+        }
+
+        public MethodInfo Method { get; }
+        public int Moment { get; }
+    }
+
+    public static class MomentLock
+    {
+        public static readonly object Lock = new();
+    }
+}
--- a/ContinuousTests/TestMomentAttribute.cs
+++ b/ContinuousTests/TestMomentAttribute.cs
@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestMomentAttribute : Attribute
+    {
+        public TestMomentAttribute(int t)
+        {
+            T = t;
+        }
+
+        public int T { get; }
+    }
+}
--- a/ContinuousTests/TestStarter.cs
+++ b/ContinuousTests/TestStarter.cs
@ -0,0 +1,36 @@
+namespace ContinuousTests
+{
+    public class TestStarter
+    {
+        private readonly Configuration config;
+        private readonly Type testType;
+        private readonly TimeSpan runsEvery;
+
+        public TestStarter(Configuration config, Type testType, TimeSpan runsEvery)
+        {
+            this.config = config;
+            this.testType = testType;
+            this.runsEvery = runsEvery;
+        }
+
+        public void Begin()
+        {
+            Task.Run(() =>
+            {
+                while (true)
+                {
+                    StartTest();
+                    Thread.Sleep(runsEvery);
+                }
+            });
+        }
+
+        private void StartTest()
+        {
+            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
+            var handle = new TestHandle(test);
+            var run = new SingleTestRun(config, handle);
+            run.Run();
+        }
+    }
+}
--- a/ContinuousTests/Tests/MarketplaceTest.cs
+++ b/ContinuousTests/Tests/MarketplaceTest.cs
@ -0,0 +1,25 @@
+using DistTestCore.Codex;
+
+namespace ContinuousTests.Tests
+{
+    //public class MarketplaceTest : ContinuousTest
+    //{
+    //    public override int RequiredNumberOfNodes => 1;
+    //    public override TimeSpan RunTestEvery => TimeSpan.FromDays(1);
+    //    public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+    //    [TestMoment(t: Zero)]
+    //    public void NodePostsStorageRequest()
+    //    {
+    //        //var c = new KubernetesWorkflow.WorkflowCreator(Log, new KubernetesWorkflow.Configuration());
+    //        //var flow = c.CreateWorkflow();
+    //        //var rc = flow.Start(10, KubernetesWorkflow.Location.Unspecified, new CodexContainerRecipe(), new KubernetesWorkflow.StartupConfig());
+    //    }
+
+    //    [TestMoment(t: DayThree)]
+    //    public void NodeDownloadsStorageRequestData()
+    //    {
+
+    //    }
+    //}
+}
--- a/ContinuousTests/Tests/PerformanceTests.cs
+++ b/ContinuousTests/Tests/PerformanceTests.cs
@ -0,0 +1,86 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class UploadPerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+
+        [TestMoment(t: Zero)]
+        public void UploadTest()
+        {
+            UploadTest(100, Nodes[0]);
+        }
+    }
+
+    public class DownloadLocalPerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
+        {
+            DownloadTest(100, Nodes[0], Nodes[0]);
+        }
+    }
+
+    public class DownloadRemotePerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
+        {
+            DownloadTest(100, Nodes[0], Nodes[1]);
+        }
+    }
+
+    public abstract class PerformanceTest : ContinuousTest
+    {
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+        public void UploadTest(int megabytes, CodexNode uploadNode)
+        {
+            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+            var time = Measure(() =>
+            {
+                UploadFile(uploadNode, file);
+            });
+
+            var timePerMB = time / megabytes;
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
+        }
+
+        public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode)
+        {
+            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+            var cid = UploadFile(uploadNode, file);
+            Assert.That(cid, Is.Not.Null);
+
+            TestFile? result = null;
+            var time = Measure(() =>
+            {
+                result = DownloadContent(downloadNode, cid!);
+            });
+
+            file.AssertIsEqual(result);
+
+            var timePerMB = time / megabytes;
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+        }
+
+        private static TimeSpan Measure(Action action)
+        {
+            var start = DateTime.UtcNow;
+            action();
+            return DateTime.UtcNow - start;
+        }
+    }
+}
--- a/ContinuousTests/Tests/TwoClientTest.cs
+++ b/ContinuousTests/Tests/TwoClientTest.cs
@ -0,0 +1,32 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class TwoClientTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            file = FileManager.GenerateTestFile(10.MB());
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+        }
+
+        [TestMoment(t: MinuteFive)]
+        public void DownloadTestFile()
+        {
+            var dl = DownloadContent(Nodes[1], cid!);
+
+            file.AssertIsEqual(dl);
+        }
+    }
+}
--- a/DistTestCore/Codex/CodexAccess.cs
+++ b/DistTestCore/Codex/CodexAccess.cs
@ -10,68 +10,19 @@ namespace DistTestCore.Codex
        {
            this.lifecycle = lifecycle;
            Container = runningContainer;
+
+            var address = lifecycle.Configuration.GetAddress(Container);
+            Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address);
        }

        public RunningContainer Container { get; }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
-        {
-            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
-        {
-            var http = Http(timeout);
-            var str = http.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
-            {
-                return new CodexDebugPeerResponse
-                {
-                    IsPeerFound = false
-                };
-            }
-
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
-            result.IsPeerFound = true;
-            return result;
-        }
-
-        public string UploadFile(FileStream fileStream)
-        {
-            return Http().HttpPostStream("upload", fileStream);
-        }
-
-        public Stream DownloadFile(string contentId)
-        {
-            return Http().HttpGetStream("download/" + contentId);
-        }
-
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
-        {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
-        }
-
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
-        {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-        }
+        public CodexNode Node { get; }

        public void EnsureOnline()
        {
            try
            {
-                var debugInfo = GetDebugInfo();
+                var debugInfo = Node.GetDebugInfo();
                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");

                var nodePeerId = debugInfo.id;
@ -85,106 +36,5 @@ namespace DistTestCore.Codex
                throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
            }
        }
-
-        private Http Http(TimeSpan? timeoutOverride = null)
-        {
-            var address = lifecycle.Configuration.GetAddress(Container);
-            return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride);
-        }
-    }
-
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set;}
    }
 }
--- a/DistTestCore/Codex/CodexContainerRecipe.cs
+++ b/DistTestCore/Codex/CodexContainerRecipe.cs
@ -50,7 +50,7 @@ namespace DistTestCore.Codex
            {
                var gethConfig = startupConfig.Get<GethStartResult>();
                var companionNode = gethConfig.CompanionNode;
-                var companionNodeAccount = companionNode.Accounts[Index];
+                var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)];
                Additional(companionNodeAccount);

                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
@ -60,7 +60,18 @@ namespace DistTestCore.Codex
                AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
                AddEnvVar("PERSISTENCE", "1");
+
+                if (config.MarketplaceConfig.IsValidator)
+                {
+                    AddEnvVar("VALIDATOR", "1");
+                }
            }
        }
+
+        private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
+        {
+            if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
+            return Index;
+        }
    }
 }
--- a/DistTestCore/Codex/CodexDeployment.cs
+++ b/DistTestCore/Codex/CodexDeployment.cs
@ -0,0 +1,45 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexDeployment
+    {
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata)
+        {
+            GethStartResult = gethStartResult;
+            CodexContainers = codexContainers;
+            Metadata = metadata;
+        }
+
+        public GethStartResult GethStartResult { get; }
+        public RunningContainer[] CodexContainers { get; }
+        public DeploymentMetadata Metadata { get; }
+    }
+
+    public class DeploymentMetadata
+    {
+        public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel)
+        {
+            DeployDateTimeUtc = DateTime.UtcNow;
+            CodexImage = codexImage;
+            GethImage = gethImage;
+            ContractsImage = contractsImage;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuotaMB = storageQuotaMB;
+            CodexLogLevel = codexLogLevel;
+        }
+        
+        public string CodexImage { get; }
+        public DateTime DeployDateTimeUtc { get; }
+        public string GethImage { get; }
+        public string ContractsImage { get; }
+        public string KubeNamespace { get; }
+        public int NumberOfCodexNodes { get; }
+        public int NumberOfValidators { get; }
+        public int StorageQuotaMB { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+    }
+}
--- a/DistTestCore/Codex/CodexNode.cs
+++ b/DistTestCore/Codex/CodexNode.cs
@ -0,0 +1,173 @@
+using Logging;
+using Utils;
+
+namespace DistTestCore.Codex
+{
+    public class CodexNode
+    {
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+
+        public CodexNode(BaseLog log, ITimeSet timeSet, Address address)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            Address = address;
+        }
+
+        public Address Address { get; }
+
+        public CodexDebugResponse GetDebugInfo()
+        {
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            var http = Http(timeout);
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
+            {
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
+            }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
+        }
+
+        public string UploadFile(FileStream fileStream)
+        {
+            return Http().HttpPostStream("upload", fileStream);
+        }
+
+        public Stream DownloadFile(string contentId)
+        {
+            return Http().HttpGetStream("download/" + contentId);
+        }
+
+        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        {
+            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+        }
+
+        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        {
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
+        }
+
+        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        {
+            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+        }
+
+        private Http Http(TimeSpan? timeoutOverride = null)
+        {
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
+        }
+    }
+
+    public class CodexDebugResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string[] addrs { get; set; } = new string[0];
+        public string repo { get; set; } = string.Empty;
+        public string spr { get; set; } = string.Empty;
+        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
+        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
+        public CodexDebugVersionResponse codex { get; set; } = new();
+        public CodexDebugTableResponse table { get; set; } = new();
+    }
+
+    public class CodexDebugTableResponse
+    {
+        public CodexDebugTableNodeResponse localNode { get; set; } = new();
+        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
+    }
+
+    public class CodexDebugTableNodeResponse
+    {
+        public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
+        public string record { get; set; } = string.Empty;
+        public string address { get; set; } = string.Empty;
+        public bool seen { get; set; }
+    }
+
+    public class EnginePeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public EnginePeerContextResponse context { get; set; } = new();
+    }
+
+    public class EnginePeerContextResponse
+    {
+        public int blocks { get; set; } = 0;
+        public int peerWants { get; set; } = 0;
+        public int exchanged { get; set; } = 0;
+        public string lastExchange { get; set; } = string.Empty;
+    }
+
+    public class SwitchPeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public string key { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugVersionResponse
+    {
+        public string version { get; set; } = string.Empty;
+        public string revision { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugPeerResponse
+    {
+        public bool IsPeerFound { get; set; }
+
+        public string peerId { get; set; } = string.Empty;
+        public long seqNo { get; set; }
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesAvailabilityRequest
+    {
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesAvailabilityResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesRequestStorageRequest
+    {
+        public string duration { get; set; } = string.Empty;
+        public string proofProbability { get; set; } = string.Empty;
+        public string reward { get; set; } = string.Empty;
+        public string collateral { get; set; } = string.Empty;
+        public string? expiry { get; set; }
+        public uint? nodes { get; set; }
+        public uint? tolerance { get; set; }
+    }
+}
--- a/DistTestCore/CodexSetup.cs
+++ b/DistTestCore/CodexSetup.cs
@ -13,6 +13,7 @@ namespace DistTestCore
        ICodexSetup EnableMetrics();
        ICodexSetup EnableMarketplace(TestToken initialBalance);
        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
    }
    
    public class CodexSetup : CodexStartupConfig, ICodexSetup
@ -62,7 +63,12 @@ namespace DistTestCore

        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
        {
-            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance);
+            return EnableMarketplace(initialBalance, initialEther, false);
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
+        {
+            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator);
            return this;
        }

--- a/DistTestCore/Configuration.cs
+++ b/DistTestCore/Configuration.cs
@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using Utils;

 namespace DistTestCore
 {
@ -16,10 +17,20 @@ namespace DistTestCore
        {
            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
-            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true";
            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
-            codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+            codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+        }
+
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
+        {
+            this.kubeConfigFile = kubeConfigFile;
+            this.logPath = logPath;
+            this.logDebug = logDebug;
+            this.dataFilesPath = dataFilesPath;
+            this.codexLogLevel = codexLogLevel;
+            this.runnerLocation = runnerLocation;
        }

        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
@ -52,7 +63,7 @@ namespace DistTestCore
            return runnerLocation;
        }

-        public RunningContainerAddress GetAddress(RunningContainer container)
+        public Address GetAddress(RunningContainer container)
        {
            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
            {
@ -74,11 +85,6 @@ namespace DistTestCore
            if (v == null) return defaultValue;
            return v;
        }
-
-        private static T ParseEnum<T>(string value)
-        {
-            return (T)Enum.Parse(typeof(T), value, true);
-        }
    }

    public enum TestRunnerLocation
--- a/DistTestCore/FileManager.cs
+++ b/DistTestCore/FileManager.cs
@ -19,11 +19,11 @@ namespace DistTestCore
        public const int ChunkSize = 1024 * 1024 * 100;
        private static NumberSource folderNumberSource = new NumberSource(0);
        private readonly Random random = new Random();
-        private readonly TestLog log;
+        private readonly BaseLog log;
        private readonly string folder;
        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();

-        public FileManager(TestLog log, Configuration configuration)
+        public FileManager(BaseLog log, Configuration configuration)
        {
            folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));

@ -142,9 +142,9 @@ namespace DistTestCore

    public class TestFile
    {
-        private readonly TestLog log;
+        private readonly BaseLog log;

-        public TestFile(TestLog log, string filename, string label)
+        public TestFile(BaseLog log, string filename, string label)
        {
            this.log = log;
            Filename = filename;
--- a/DistTestCore/GethStarter.cs
+++ b/DistTestCore/GethStarter.cs
@ -33,6 +33,8 @@ namespace DistTestCore

        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
        {
+            if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
+
            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
            var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;

--- a/DistTestCore/Http.cs
+++ b/DistTestCore/Http.cs
@ -1,5 +1,4 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 using Newtonsoft.Json;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
@ -11,11 +10,11 @@ namespace DistTestCore
    {
        private readonly BaseLog log;
        private readonly ITimeSet timeSet;
-        private readonly RunningContainerAddress address;
+        private readonly Address address;
        private readonly string baseUrl;
        private readonly TimeSpan? timeoutOverride;

-        public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null)
        {
            this.log = log;
            this.timeSet = timeSet;
--- a/DistTestCore/Marketplace/GethStartResult.cs
+++ b/DistTestCore/Marketplace/GethStartResult.cs
@ -1,4 +1,6 @@
-namespace DistTestCore.Marketplace
+using Newtonsoft.Json;
+
+namespace DistTestCore.Marketplace
 {
    public class GethStartResult
    {
@ -9,6 +11,7 @@
            CompanionNode = companionNode;
        }

+        [JsonIgnore]
        public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
        public MarketplaceNetwork MarketplaceNetwork { get; }
        public GethCompanionNodeInfo CompanionNode { get; }
--- a/DistTestCore/Marketplace/MarketplaceAccess.cs
+++ b/DistTestCore/Marketplace/MarketplaceAccess.cs
@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
                $"proofProbability: {proofProbability}, " +
                $"duration: {Time.FormatDuration(duration)})");

-            var response = codexAccess.RequestStorage(request, contentId.Id);
+            var response = codexAccess.Node.RequestStorage(request, contentId.Id);

            if (response == "Purchasing not available")
            {
@ -78,7 +78,7 @@ namespace DistTestCore.Marketplace
                $"maxCollateral: {maxCollateral}, " +
                $"maxDuration: {Time.FormatDuration(maxDuration)})");

-            var response = codexAccess.SalesAvailability(request);
+            var response = codexAccess.Node.SalesAvailability(request);

            Log($"Storage successfully made available. Id: {response.id}");

--- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs
+++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs
@ -2,13 +2,16 @@
 {
    public class MarketplaceInitialConfig
    {
-        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens)
+        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
        {
            InitialEth = initialEth;
            InitialTestTokens = initialTestTokens;
+            IsValidator = isValidator;
        }

        public Ether InitialEth { get; }
        public TestToken InitialTestTokens { get; }
+        public bool IsValidator { get; }
+        public int? AccountIndexOverride { get; set; }
    }
 }
--- a/DistTestCore/OnlineCodexNode.cs
+++ b/DistTestCore/OnlineCodexNode.cs
@ -49,7 +49,7 @@ namespace DistTestCore

        public CodexDebugResponse GetDebugInfo()
        {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = CodexAccess.Node.GetDebugInfo();
            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
            return debugInfo;
@ -57,12 +57,12 @@ namespace DistTestCore

        public CodexDebugPeerResponse GetDebugPeer(string peerId)
        {
-            return CodexAccess.GetDebugPeer(peerId);
+            return CodexAccess.Node.GetDebugPeer(peerId);
        }

        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
        {
-            return CodexAccess.GetDebugPeer(peerId, timeout);
+            return CodexAccess.Node.GetDebugPeer(peerId, timeout);
        }

        public ContentId UploadFile(TestFile file)
@ -72,7 +72,7 @@ namespace DistTestCore
            var logMessage = $"Uploading file {file.Describe()}...";
            var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
            {
-                return CodexAccess.UploadFile(fileStream);
+                return CodexAccess.Node.UploadFile(fileStream);
            });

            if (response.StartsWith(UploadFailedMessage))
@ -101,7 +101,7 @@ namespace DistTestCore

            Log($"Connecting to peer {peer.GetName()}...");
            var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));

            Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
            Log($"Successfully connected to peer {peer.GetName()}.");
@ -141,7 +141,7 @@ namespace DistTestCore
            using var fileStream = File.OpenWrite(file.Filename);
            try
            {
-                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                using var downloadStream = CodexAccess.Node.DownloadFile(contentId);
                downloadStream.CopyTo(fileStream);
            }
            catch
--- a/DistTestCore/TestLifecycle.cs
+++ b/DistTestCore/TestLifecycle.cs
@ -7,15 +7,18 @@ namespace DistTestCore
 {
    public class TestLifecycle
    {
-        private readonly WorkflowCreator workflowCreator;
        private DateTime testStart = DateTime.MinValue;

        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
+            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
+        {
+        }
+
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
        {
            Log = log;
            Configuration = configuration;
            TimeSet = timeSet;
-            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));

            FileManager = new FileManager(Log, configuration);
            CodexStarter = new CodexStarter(this, workflowCreator);
--- a/KubernetesWorkflow/RunningContainers.cs
+++ b/KubernetesWorkflow/RunningContainers.cs
@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Utils;
+
+namespace KubernetesWorkflow
 {
    public class RunningContainers
    {
@ -21,12 +23,12 @@

    public class RunningContainer
    {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
        {
            Pod = pod;
            Recipe = recipe;
            ServicePorts = servicePorts;
-            Name = GetContainerName(recipe, startupConfig);
+            Name = name;
            ClusterExternalAddress = clusterExternalAddress;
            ClusterInternalAddress = clusterInternalAddress;
        }
@ -35,31 +37,7 @@
        public RunningPod Pod { get; }
        public ContainerRecipe Recipe { get; }
        public Port[] ServicePorts { get; }
-        public RunningContainerAddress ClusterExternalAddress { get; }
-        public RunningContainerAddress ClusterInternalAddress { get; }
-
-        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
-        {
-            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
-            {
-                return $"<{startupConfig.NameOverride}{recipe.Number}>";
-            }
-            else
-            {
-                return $"<{recipe.Name}>";
-            }
-        }
-    }
-
-    public class RunningContainerAddress
-    {
-        public RunningContainerAddress(string host, int port)
-        {
-            Host = host;
-            Port = port;
-        }
-
-        public string Host { get; }
-        public int Port { get; }
+        public Address ClusterExternalAddress { get; }
+        public Address ClusterInternalAddress { get; }
    }
 }
--- a/KubernetesWorkflow/StartupWorkflow.cs
+++ b/KubernetesWorkflow/StartupWorkflow.cs
@ -1,4 +1,5 @@
 using Logging;
+using Utils;

 namespace KubernetesWorkflow
 {
@ -80,27 +81,42 @@ namespace KubernetesWorkflow
                var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
                log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");

-                return new RunningContainer(runningPod, r, servicePorts, startupConfig,
+                var name = GetContainerName(r, startupConfig);
+
+                return new RunningContainer(runningPod, r, servicePorts, name,
                    GetContainerExternalAddress(runningPod, servicePorts),
                    GetContainerInternalAddress(r));

            }).ToArray();
        }

-        private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
        {
-            return new RunningContainerAddress(
+            if (startupConfig == null) return "";
+            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
+            {
+                return $"<{startupConfig.NameOverride}{recipe.Number}>";
+            }
+            else
+            {
+                return $"<{recipe.Name}>";
+            }
+        }
+
+        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        {
+            return new Address(
                pod.Cluster.HostAddress,
                GetServicePort(servicePorts));
        }

-        private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe)
+        private Address GetContainerInternalAddress(ContainerRecipe recipe)
        {
            var serviceName = "service-" + numberSource.WorkflowNumber;
            var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
            var port = GetInternalPort(recipe);

-            return new RunningContainerAddress(
+            return new Address(
                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                port);
        }
--- a/KubernetesWorkflow/WorkflowCreator.cs
+++ b/KubernetesWorkflow/WorkflowCreator.cs
@ -13,10 +13,15 @@ namespace KubernetesWorkflow
        private readonly string testNamespace;

        public WorkflowCreator(BaseLog log, Configuration configuration)
+            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
+        {
+        }
+
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
        {
            cluster = new K8sCluster(configuration);
            this.log = log;
-            testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
+            testNamespace = testNamespacePostfix;
        }

        public StartupWorkflow CreateWorkflow()
--- a/Logging/BaseLog.cs
+++ b/Logging/BaseLog.cs
@ -25,12 +25,12 @@ namespace Logging
            }
        }

-        public void Log(string message)
+        public virtual void Log(string message)
        {
            LogFile.Write(ApplyReplacements(message));
        }

-        public void Debug(string message = "", int skipFrames = 0)
+        public virtual void Debug(string message = "", int skipFrames = 0)
        {
            if (debug)
            {
@ -40,24 +40,29 @@ namespace Logging
            }
        }

-        public void Error(string message)
+        public virtual void Error(string message)
        {
            Log($"[ERROR] {message}");
        }

-        public void MarkAsFailed()
+        public virtual void MarkAsFailed()
        {
            if (hasFailed) return;
            hasFailed = true;
            LogFile.ConcatToFilename("_FAILED");
        }

-        public void AddStringReplace(string from, string to)
+        public virtual void AddStringReplace(string from, string to)
        {
            if (string.IsNullOrWhiteSpace(from)) return;
            replacements.Add(new BaseLogStringReplacement(from, to));
        }

+        public virtual void Delete()
+        {
+            File.Delete(LogFile.FullFilename);
+        }
+
        private string ApplyReplacements(string str)
        {
            foreach (var replacement in replacements)
--- a/Logging/FixtureLog.cs
+++ b/Logging/FixtureLog.cs
@ -8,19 +8,24 @@ namespace Logging
        private readonly string fullName;
        private readonly LogConfig config;

-        public FixtureLog(LogConfig config)
+        public FixtureLog(LogConfig config, string name = "")
            : base(config.DebugEnabled)
        {
            start = DateTime.UtcNow;
            var folder = DetermineFolder(config);
-            var fixtureName = GetFixtureName();
+            var fixtureName = GetFixtureName(name);
            fullName = Path.Combine(folder, fixtureName);
            this.config = config;
        }

-        public TestLog CreateTestLog()
+        public TestLog CreateTestLog(string name = "")
        {
-            return new TestLog(fullName, config.DebugEnabled);
+            return new TestLog(fullName, config.DebugEnabled, name);
+        }
+
+        public void DeleteFolder()
+        {
+            Directory.Delete(fullName, true);
        }

        protected override LogFile CreateLogFile()
@ -36,10 +41,12 @@ namespace Logging
               Pad(start.Day));
        }

-        private string GetFixtureName()
+        private string GetFixtureName(string name)
        {
            var test = TestContext.CurrentContext.Test;
            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            if (!string.IsNullOrEmpty(name)) className = name;
+
            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
        }

--- a/Logging/TestLog.cs
+++ b/Logging/TestLog.cs
@ -9,10 +9,10 @@ namespace Logging
        private readonly string methodName;
        private readonly string fullName;

-        public TestLog(string folder, bool debug)
+        public TestLog(string folder, bool debug, string name = "")
            : base(debug)
        {
-            methodName = GetMethodName();
+            methodName = GetMethodName(name);
            fullName = Path.Combine(folder, methodName);

            Log($"*** Begin: {methodName}");
@ -39,13 +39,15 @@ namespace Logging
                MarkAsFailed();
            }
        }
+
        protected override LogFile CreateLogFile()
        {
            return new LogFile(fullName, "log");
        }

-        private string GetMethodName()
+        private string GetMethodName(string name)
        {
+            if (!string.IsNullOrEmpty(name)) return name;
            var test = TestContext.CurrentContext.Test;
            var args = FormatArguments(test);
            return $"{test.MethodName}{args}";
--- a/Utils/Address.cs
+++ b/Utils/Address.cs
@ -0,0 +1,14 @@
+namespace Utils
+{
+    public class Address
+    {
+        public Address(string host, int port)
+        {
+            Host = host;
+            Port = port;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+    }
+}
--- a/Utils/ParseEnum.cs
+++ b/Utils/ParseEnum.cs
@ -0,0 +1,10 @@
+namespace Utils
+{
+    public static class ParseEnum
+    {
+        public static T Parse<T>(string value)
+        {
+            return (T)Enum.Parse(typeof(T), value, true);
+        }
+    }
+}
--- a/Utils/RandomUtils.cs
+++ b/Utils/RandomUtils.cs
@ -0,0 +1,15 @@
+namespace Utils
+{
+    public static class RandomUtils
+    {
+        private static readonly Random random = new Random();
+
+        public static T PickOneRandom<T>(this List<T> remainingItems)
+        {
+            var i = random.Next(0, remainingItems.Count);
+            var result = remainingItems[i];
+            remainingItems.RemoveAt(i);
+            return result;
+        }
+    }
+}
--- a/cs-codex-dist-testing.sln
+++ b/cs-codex-dist-testing.sln
@ -17,6 +17,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@ -51,6 +55,14 @@ Global
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE