Merge branch 'feature/continuous-testing'

CodexNetDeployer/ArgOrVar.cs

@@ -0,0 +1,93 @@
+namespace CodexNetDeployer
+{
+    public class ArgOrVar
+    {
+        public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex.");
+        public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth.");
+        public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer.");
+        public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file.");
+        public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment.");
+        public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created.");
+        public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating.");
+        public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node.");
+        public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]");
+
+        private readonly string[] args;
+
+        public ArgOrVar(string[] args)
+        {
+            this.args = args;
+        }
+
+        public string Get(ArgVar key, string defaultValue = "")
+        {
+            var argKey = $"--{key.Arg}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+
+            var env = Environment.GetEnvironmentVariable(key.Var);
+            if (env != null)
+            {
+                return env;
+            }
+
+            return defaultValue;
+        }
+        
+        public int? GetInt(ArgVar key)
+        {
+            var str = Get(key);
+            if (string.IsNullOrEmpty(str)) return null;
+            if (int.TryParse(str, out int result))
+            {
+                return result;
+            }
+            return null;
+        }
+
+        public void PrintHelp()
+        {
+            var nl = Environment.NewLine;
+            Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+                "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
+                "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+            Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+
+            Console.Write("\t[ CLI argument ] or [ Environment variable ]");
+            Console.CursorLeft = 70;
+            Console.Write("(Description)" + nl);
+            var fields = GetType().GetFields();
+            foreach (var field in fields)
+            {
+                var value = (ArgVar)field.GetValue(null)!;
+                value.PrintHelp();
+            }
+        }
+    }
+
+    public class ArgVar
+    {
+        public ArgVar(string arg, string var, string description)
+        {
+            Arg = arg;
+            Var = var;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string Var { get; }
+        public string Description { get; }
+
+        public void PrintHelp()
+        {
+            Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]");
+            Console.CursorLeft = 70;
+            Console.Write(Description + Environment.NewLine);
+        }
+    }
+}

CodexNetDeployer/CodexNetDeployer.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDeployer/CodexNodeStarter.cs

@@ -0,0 +1,75 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class CodexNodeStarter
+    {
+        private readonly Configuration config;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly TestLifecycle lifecycle;
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+        private readonly GethStartResult gethResult;
+        private string bootstrapSpr = "";
+        private int validatorsLeft;
+
+        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators)
+        {
+            this.config = config;
+            this.workflowCreator = workflowCreator;
+            this.lifecycle = lifecycle;
+            this.log = log;
+            this.timeSet = timeSet;
+            this.gethResult = gethResult;
+            this.validatorsLeft = numberOfValidators;
+        }
+
+        public RunningContainer? Start(int i)
+        {
+            Console.Write($" - {i} = ");
+            var workflow = workflowCreator.CreateWorkflow();
+            var workflowStartup = new StartupConfig();
+            workflowStartup.Add(gethResult);
+            workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
+
+            var container = containers.Containers.First();
+            var address = lifecycle.Configuration.GetAddress(container);
+            var codexNode = new CodexNode(log, timeSet, address);
+            var debugInfo = codexNode.GetDebugInfo();
+
+            if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+            {
+                var pod = container.Pod.PodInfo;
+                Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine);
+
+                if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+                validatorsLeft--;
+                return container;
+            }
+            else
+            {
+                Console.Write("Unknown failure." + Environment.NewLine);
+                return null;
+            }
+        }
+
+        private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
+        {
+            var codexStart = new CodexStartupConfig(config.CodexLogLevel);
+
+            if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
+            codexStart.StorageQuota = config.StorageQuota!.Value.MB();
+            var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
+            marketplaceConfig.AccountIndexOverride = i;
+            codexStart.MarketplaceConfig = marketplaceConfig;
+
+            return codexStart;
+        }
+    }
+}

CodexNetDeployer/Configuration.cs

@@ -0,0 +1,101 @@
+using DistTestCore;
+using DistTestCore.Codex;
+
+namespace CodexNetDeployer
+{
+    public class Configuration
+    {
+        public Configuration(
+            string codexImage,
+            string gethImage,
+            string contractsImage,
+            string kubeConfigFile,
+            string kubeNamespace,
+            int? numberOfCodexNodes,
+            int? numberOfValidators,
+            int? storageQuota,
+            CodexLogLevel codexLogLevel,
+            TestRunnerLocation runnerLocation)
+        {
+            CodexImage = codexImage;
+            GethImage = gethImage;
+            ContractsImage = contractsImage;
+            KubeConfigFile = kubeConfigFile;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuota = storageQuota;
+            CodexLogLevel = codexLogLevel;
+            RunnerLocation = runnerLocation;
+        }
+
+        public string CodexImage { get; }
+        public string GethImage { get; }
+        public string ContractsImage { get; }
+        public string KubeConfigFile { get; }
+        public string KubeNamespace { get; }
+        public int? NumberOfCodexNodes { get; }
+        public int? NumberOfValidators { get; }
+        public int? StorageQuota { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+        public TestRunnerLocation RunnerLocation { get; }
+
+        public void PrintConfig()
+        {
+            ForEachProperty(onString: Print, onInt: Print);
+        }
+
+        public List<string> Validate()
+        {
+            var errors = new List<string>();
+
+            ForEachProperty(
+                onString: (n, v) => StringIsSet(n, v, errors),
+                onInt: (n, v) => IntIsOverZero(n, v, errors));
+
+            if (NumberOfValidators > NumberOfCodexNodes)
+            {
+                errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
+            }
+
+            return errors;
+        }
+
+        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
+        {
+            var properties = GetType().GetProperties();
+            foreach (var p in properties)
+            {
+                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
+            }
+        }
+
+        private static void IntIsOverZero(string variable, int? value, List<string> errors)
+        {
+            if (value == null || value.Value < 1)
+            {
+                errors.Add($"{variable} is must be set and must be greater than 0.");
+            }
+        }
+
+        private static void StringIsSet(string variable, string value, List<string> errors)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                errors.Add($"{variable} is must be set.");
+            }
+        }
+
+        private static void Print(string variable, string value)
+        {
+            Console.WriteLine($"\t{variable}: '{value}'");
+        }
+
+        private static void Print(string variable, int? value)
+        {
+            if (value != null) Print(variable, value.ToString()!);
+            else Print(variable, "<NONE>");
+        }
+    }
+}

CodexNetDeployer/Deployer.cs

@@ -0,0 +1,93 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace CodexNetDeployer
+{
+    public class Deployer
+    {
+        private readonly Configuration config;
+        private readonly NullLog log;
+        private readonly DefaultTimeSet timeset;
+
+        public Deployer(Configuration config)
+        {
+            this.config = config;
+            log = new NullLog();
+            timeset = new DefaultTimeSet();
+        }
+
+        public CodexDeployment Deploy()
+        {
+            Log("Initializing...");
+            var (workflowCreator, lifecycle) = CreateFacilities();
+
+            Log("Preparing configuration...");
+            // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
+            var setup = new CodexSetup(999, config.CodexLogLevel);
+            setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+
+            Log("Creating Geth instance and deploying contracts...");
+            var gethStarter = new GethStarter(lifecycle, workflowCreator);
+            var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
+
+            Log("Geth started. Codex contracts deployed.");
+            Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
+
+            Log("Starting Codex nodes...");
+
+            // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
+            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value);
+            var codexContainers = new List<RunningContainer>();
+            for (var i = 0; i < config.NumberOfCodexNodes; i++)
+            {
+                var container = codexStarter.Start(i);
+                if (container != null) codexContainers.Add(container);
+            }
+
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata());
+        }
+
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        {
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: config.KubeConfigFile,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: "notUsed",
+                codexLogLevel: config.CodexLogLevel,
+                runnerLocation: config.RunnerLocation
+            );
+
+            var kubeConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: config.KubeNamespace,
+                kubeConfigFile: config.KubeConfigFile,
+                operationTimeout: timeset.K8sOperationTimeout(),
+                retryDelay: timeset.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
+
+            return (workflowCreator, lifecycle);
+        }
+
+        private DeploymentMetadata CreateMetadata()
+        {
+            return new DeploymentMetadata(
+                codexImage: config.CodexImage,
+                gethImage: config.GethImage,
+                contractsImage: config.ContractsImage,
+                kubeNamespace: config.KubeNamespace,
+                numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
+                numberOfValidators: config.NumberOfValidators!.Value,
+                storageQuotaMB: config.StorageQuota!.Value,
+                codexLogLevel: config.CodexLogLevel);
+        }
+
+        private void Log(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+    }
+}

CodexNetDeployer/NullLog.cs

@@ -0,0 +1,43 @@
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class NullLog : TestLog
+    {
+        public NullLog() : base("NULL", false, "NULL")
+        {
+        }
+
+        protected override LogFile CreateLogFile()
+        {
+            return null!;
+        }
+
+        public override void Log(string message)
+        {
+            //Console.WriteLine(message);
+        }
+
+        public override void Debug(string message = "", int skipFrames = 0)
+        {
+            //Console.WriteLine(message);
+        }
+
+        public override void Error(string message)
+        {
+            Console.WriteLine("Error: " + message);
+        }
+
+        public override void MarkAsFailed()
+        {
+        }
+
+        public override void AddStringReplace(string from, string to)
+        {
+        }
+
+        public override void Delete()
+        {
+        }
+    }
+}

CodexNetDeployer/Program.cs

@@ -0,0 +1,66 @@
+using CodexNetDeployer;
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using Newtonsoft.Json;
+using Utils;
+using Configuration = CodexNetDeployer.Configuration;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDeployer" + nl + nl);
+
+        var argOrVar = new ArgOrVar(args);
+
+        if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+        {
+            argOrVar.PrintHelp();
+            return;
+        }
+
+        var location = TestRunnerLocation.InternalToCluster;
+        if (args.Any(a => a == "--external"))
+        {
+            location = TestRunnerLocation.ExternalToCluster;
+        }
+
+        var config = new Configuration(
+            codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage),
+            gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage),
+            contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage),
+            kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile),
+            kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace),
+            numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes),
+            numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes),
+            storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota),
+            codexLogLevel: ParseEnum.Parse<CodexLogLevel>(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))),
+            runnerLocation: location
+        );
+
+        Console.WriteLine("Using:");
+        config.PrintConfig();
+        Console.WriteLine(nl);
+
+        var errors = config.Validate();
+        if (errors.Any())
+        {
+            Console.WriteLine($"Configuration errors: ({errors.Count})");
+            foreach ( var error in errors ) Console.WriteLine("\t" + error);
+            Console.WriteLine(nl);
+            argOrVar.PrintHelp();
+            return;
+        }
+
+        var deployer = new Deployer(config);
+        var deployment = deployer.Deploy();
+
+        Console.WriteLine("Writing codex-deployment.json...");
+
+        File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented));
+
+        Console.WriteLine("Done!");
+    }
+}

ContinuousTests/CodexNodeFactory.cs

@@ -0,0 +1,19 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class CodexNodeFactory
+    {
+        public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+        {
+            return containers.Select(container =>
+            {
+                var address = container.ClusterExternalAddress;
+                return new CodexNode(log, timeSet, address);
+            }).ToArray();
+        }
+    }
+}

ContinuousTests/Configuration.cs

@@ -0,0 +1,98 @@
+using DistTestCore.Codex;
+using Newtonsoft.Json;
+
+namespace ContinuousTests
+{
+    public class Configuration
+    {
+        public string LogPath { get; set; } = string.Empty;
+        public string DataPath { get; set; } = string.Empty;
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+        public bool KeepPassedTestLogs { get; set; }
+    }
+
+    public class ConfigLoader
+    {
+        private const string filename = "config.json";
+
+        public Configuration Load()
+        {
+            var config = Read();
+
+            Validate(config);
+            return config;
+        }
+
+        private Configuration Read()
+        {
+            if (File.Exists(filename))
+            {
+                var lines = File.ReadAllText(filename);
+                try
+                {
+                    var result = JsonConvert.DeserializeObject<Configuration>(lines);
+                    if (result != null) return result;
+                }
+                catch { }
+            }
+
+            var logPath = Environment.GetEnvironmentVariable("LOGPATH");
+            var dataPath = Environment.GetEnvironmentVariable("DATAPATH");
+            var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT");
+            var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS");
+
+            if (!string.IsNullOrEmpty(logPath) &&
+                !string.IsNullOrEmpty(dataPath) &&
+                !string.IsNullOrEmpty(codexDeploymentJson))
+            {
+                try 
+                {
+                    return new Configuration
+                    { 
+                        LogPath = logPath,
+                        DataPath = dataPath,
+                        CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson),
+                        KeepPassedTestLogs = keep == "1"
+                    };
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine("Exception: " + ex);
+                }
+            }
+
+            var nl = Environment.NewLine;
+            throw new Exception($"Unable to load configuration from '{filename}', and " +
+                "unable to load configuration from environment variables. " + nl +
+                "'LOGPATH' = Path where log files will be saved." + nl +
+                "'DATAPATH' = Path where temporary data files will be saved." + nl +
+                "'CODEXDEPLOYMENT' = Path to codex-deployment JSON file." + nl +
+                nl);
+        }
+
+        private void Validate(Configuration configuration)
+        {
+            if (string.IsNullOrEmpty(configuration.LogPath))
+            {
+                throw new Exception($"Invalid LogPath set: '{configuration.LogPath}'");
+            }
+
+            if (string.IsNullOrEmpty(configuration.DataPath))
+            {
+                throw new Exception($"Invalid DataPath set: '{configuration.DataPath}'");
+            }
+
+            if (configuration.CodexDeployment == null || !configuration.CodexDeployment.CodexContainers.Any())
+            {
+                throw new Exception("No Codex deployment found.");
+            }
+        }
+
+        private CodexDeployment ParseCodexDeploymentJson(string filename)
+        {
+            var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+            if (d == null) throw new Exception("Unable to parse " + filename);
+            return d;
+        }
+    }
+}

ContinuousTests/ContinuousTest.cs

@@ -0,0 +1,95 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+
+namespace ContinuousTests
+{
+    public abstract class ContinuousTestLongTimeouts : ContinuousTest
+    {
+        public override ITimeSet TimeSet => new LongTimeSet();
+    }
+
+    public abstract class ContinuousTest
+    {
+        protected const int Zero = 0;
+        protected const int MinuteOne = 60;
+        protected const int MinuteFive = MinuteOne * 5;
+        protected const int HourOne = MinuteOne * 60;
+        protected const int DayOne = HourOne * 24;
+        protected const int DayThree = DayOne * 3;
+
+        private const string UploadFailedMessage = "Unable to store block";
+
+        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager)
+        {
+            Nodes = nodes;
+            Log = log;
+            FileManager = fileManager;
+        }
+
+        public CodexNode[] Nodes { get; private set; } = null!;
+        public BaseLog Log { get; private set; } = null!;
+        public IFileManager FileManager { get; private set; } = null!;
+        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+
+        public abstract int RequiredNumberOfNodes { get; }
+        public abstract TimeSpan RunTestEvery { get; }
+        public abstract TestFailMode TestFailMode { get; }
+
+        public string Name
+        {
+            get
+            {
+                return GetType().Name;
+            }
+        }
+
+        public ContentId? UploadFile(CodexNode node, TestFile file)
+        {
+            using var fileStream = File.OpenRead(file.Filename);
+
+            var logMessage = $"Uploading file {file.Describe()}...";
+            var response = Stopwatch.Measure(Log, logMessage, () =>
+            {
+                return node.UploadFile(fileStream);
+            });
+
+            if (response.StartsWith(UploadFailedMessage))
+            {
+                return null;
+            }
+            Log.Log($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
+        {
+            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            var file = FileManager.CreateEmptyTestFile(fileLabel);
+            Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
+            Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+            return file;
+        }
+
+        private void DownloadToFile(CodexNode node, string contentId, TestFile file)
+        {
+            using var fileStream = File.OpenWrite(file.Filename);
+            try
+            {
+                using var downloadStream = node.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log.Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
+        }
+    }
+
+    public enum TestFailMode
+    {
+        StopAfterFirstFailure,
+        AlwaysRunAllMoments
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -0,0 +1,31 @@
+namespace ContinuousTests
+{
+    public class ContinuousTestRunner
+    {
+        private readonly ConfigLoader configLoader = new ConfigLoader();
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly Configuration config;
+        private readonly StartupChecker startupChecker;
+
+        public ContinuousTestRunner()
+        {
+            config = configLoader.Load();
+            startupChecker = new StartupChecker(config);
+        }
+
+        public void Run()
+        {
+            startupChecker.Check();
+
+            var allTests = testFactory.CreateTests();
+            var testStarters = allTests.Select(t => new TestStarter(config, t.GetType(), t.RunTestEvery)).ToArray();
+
+            foreach (var t in testStarters)
+            {
+                t.Begin();
+            }
+          
+            while (true) Thread.Sleep((2 ^ 31) - 1);
+        }
+    }
+}

ContinuousTests/ContinuousTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

ContinuousTests/Program.cs

@@ -0,0 +1,12 @@
+using ContinuousTests;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Running...");
+        var runner = new ContinuousTestRunner();
+        runner.Run();
+    }
+}

ContinuousTests/SingleTestRun.cs

@@ -0,0 +1,143 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+using Utils;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+    public class SingleTestRun
+    {
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly List<Exception> exceptions = new List<Exception>();
+        private readonly Configuration config;
+        private readonly TestHandle handle;
+        private readonly CodexNode[] nodes;
+        private readonly FileManager fileManager;
+        private readonly FixtureLog fixtureLog;
+        private readonly string dataFolder;
+
+        public SingleTestRun(Configuration config, TestHandle handle)
+        {
+            this.config = config;
+            this.handle = handle;
+
+            var testName = handle.Test.GetType().Name;
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
+
+            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            dataFolder = config.DataPath + "-" + Guid.NewGuid();
+            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
+        }
+
+        public void Run()
+        {
+            Task.Run(() =>
+            {
+                try
+                {
+                    RunTest();
+
+                    if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+                }
+                catch (Exception ex)
+                {
+                    fixtureLog.Error("Test run failed with exception: " + ex);
+                    fixtureLog.MarkAsFailed();
+                }
+                fileManager.DeleteAllTestFiles();
+                Directory.Delete(dataFolder, true);
+            });
+        }
+
+        private void RunTest()
+        {
+            var earliestMoment = handle.GetEarliestMoment();
+
+            var t = earliestMoment;
+            while (true)
+            {
+                RunMoment(t);
+
+                if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
+                {
+                    Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
+                    throw exceptions.Single();
+                }
+
+                var nextMoment = handle.GetNextMoment(t);
+                if (nextMoment != null)
+                {
+                    Log($" > Next TestMoment in {nextMoment.Value} seconds...");
+                    t += nextMoment.Value;
+                    Thread.Sleep(nextMoment.Value * 1000);
+                }
+                else
+                {
+                    if (exceptions.Any())
+                    {
+                        Log(" > Completed last test moment. Test failed.");
+                        throw exceptions.First();
+                    }
+                    Log(" > Completed last test moment. Test passed.");
+                    return;
+                }
+            }
+        }
+
+        private void RunMoment(int t)
+        {
+            try
+            {
+                handle.InvokeMoment(t, InitializeTest);
+            }
+            catch (Exception ex)
+            {
+                Log($" > TestMoment yielded exception: " + ex);
+                exceptions.Add(ex);
+            }
+
+            DecommissionTest();
+        }
+
+        private void InitializeTest(string name)
+        {
+            Log($" > Running TestMoment '{name}'");
+            handle.Test.Initialize(nodes, fixtureLog, fileManager);
+        }
+
+        private void DecommissionTest()
+        {
+            handle.Test.Initialize(null!, null!, null!);
+        }
+
+        private void Log(string msg)
+        {
+            fixtureLog.Log(msg);
+        }
+
+        private CodexNode[] CreateRandomNodes(int number)
+        {
+            var containers = SelectRandomContainers(number);
+            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
+            return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
+        }
+
+        private RunningContainer[] SelectRandomContainers(int number)
+        {
+            var containers = config.CodexDeployment.CodexContainers.ToList();
+            var result = new RunningContainer[number];
+            for (var i = 0; i < number; i++)
+            {
+                result[i] = containers.PickOneRandom();
+            }
+            return result;
+        }
+
+        private DistTestCore.Configuration CreateFileManagerConfiguration()
+        {
+            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+                CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
+        }
+    }
+}

ContinuousTests/StartupChecker.cs

@@ -0,0 +1,102 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class StartupChecker
+    {
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly Configuration config;
+
+        public StartupChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Check()
+        {
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+            log.Log("Starting continuous test run...");
+            log.Log("Checking configuration...");
+            PreflightCheck(config);
+            log.Log("Contacting Codex nodes...");
+            CheckCodexNodes(log, config);
+            log.Log("All OK.");
+        }
+
+        private void PreflightCheck(Configuration config)
+        {
+            var tests = testFactory.CreateTests();
+            if (!tests.Any())
+            {
+                throw new Exception("Unable to find any tests.");
+            }
+            foreach (var test in tests)
+            {
+                var handle = new TestHandle(test);
+                handle.GetEarliestMoment();
+                handle.GetLastMoment();
+            }
+
+            var errors = new List<string>();
+            foreach (var test in tests)
+            {
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                {
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                }
+            }
+
+            if (!Directory.Exists(config.LogPath))
+            {
+                Directory.CreateDirectory(config.LogPath);
+            }
+
+            if (errors.Any())
+            {
+                throw new Exception("Prerun check failed: " + string.Join(", ", errors));
+            }
+        }
+
+        private void CheckCodexNodes(BaseLog log, Configuration config)
+        {
+            var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+            var pass = true;
+            foreach (var n in nodes)
+            {
+                log.Log($"Checking '{n.Address.Host}'...");
+
+                if (EnsureOnline(n))
+                {
+                    log.Log("OK");
+                }
+                else
+                {
+                    log.Error($"No response from '{n.Address.Host}'.");
+                    pass = false;
+                }
+            }
+            if (!pass)
+            {
+                throw new Exception("Not all codex nodes responded.");
+            }
+        }
+
+        private bool EnsureOnline(CodexNode n)
+        {
+            try
+            {
+                var info = n.GetDebugInfo();
+                if (info == null || string.IsNullOrEmpty(info.id)) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+    }
+}

ContinuousTests/TestFactory.cs

@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestFactory
+    {
+        public ContinuousTest[] CreateTests()
+        {
+            var types = GetType().Assembly.GetTypes();
+            var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract);
+            return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray();
+        }
+    }
+}

ContinuousTests/TestHandle.cs

@@ -0,0 +1,86 @@
+using System.Reflection;
+
+namespace ContinuousTests
+{
+    public class TestHandle
+    {
+        private readonly List<MethodMoment> moments = new List<MethodMoment>();
+
+        public TestHandle(ContinuousTest test)
+        {
+            Test = test;
+
+            ReflectTestMoments();
+
+            var testName = test.GetType().Name;
+            if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
+            if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
+        }
+
+        public ContinuousTest Test { get; }
+
+        public int GetEarliestMoment()
+        {
+            return moments.Min(m => m.Moment);
+        }
+
+        public int GetLastMoment()
+        {
+            return moments.Max(m => m.Moment);
+        }
+
+        public int? GetNextMoment(int currentMoment)
+        {
+            var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
+            if (!remainingMoments.Any()) return null;
+            return remainingMoments.Min(m => m.Moment);
+        }
+
+        public void InvokeMoment(int currentMoment, Action<string> beforeInvoke)
+        {
+            var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
+            if (moment == null) return;
+
+            lock (MomentLock.Lock)
+            {
+                beforeInvoke(moment.Method.Name);
+                moment.Method.Invoke(Test, Array.Empty<object>());
+            }
+        }
+
+        private void ReflectTestMoments()
+        {
+            var methods = Test.GetType().GetMethods()
+                      .Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0)
+                      .ToArray();
+
+            foreach (var method in methods)
+            {
+                var moment = method.GetCustomAttribute<TestMomentAttribute>();
+                if (moment != null && moment.T >= 0)
+                {
+                    moments.Add(new MethodMoment(method, moment.T));
+                }
+            }
+        }
+    }
+
+    public class MethodMoment
+    {
+        public MethodMoment(MethodInfo method, int moment)
+        {
+            Method = method;
+            Moment = moment;
+
+            if (moment < 0) throw new Exception("Moment must be zero or greater.");
+        }
+
+        public MethodInfo Method { get; }
+        public int Moment { get; }
+    }
+
+    public static class MomentLock
+    {
+        public static readonly object Lock = new();
+    }
+}

ContinuousTests/TestMomentAttribute.cs

@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestMomentAttribute : Attribute
+    {
+        public TestMomentAttribute(int t)
+        {
+            T = t;
+        }
+
+        public int T { get; }
+    }
+}

ContinuousTests/TestStarter.cs

@@ -0,0 +1,36 @@
+namespace ContinuousTests
+{
+    public class TestStarter
+    {
+        private readonly Configuration config;
+        private readonly Type testType;
+        private readonly TimeSpan runsEvery;
+
+        public TestStarter(Configuration config, Type testType, TimeSpan runsEvery)
+        {
+            this.config = config;
+            this.testType = testType;
+            this.runsEvery = runsEvery;
+        }
+
+        public void Begin()
+        {
+            Task.Run(() =>
+            {
+                while (true)
+                {
+                    StartTest();
+                    Thread.Sleep(runsEvery);
+                }
+            });
+        }
+
+        private void StartTest()
+        {
+            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
+            var handle = new TestHandle(test);
+            var run = new SingleTestRun(config, handle);
+            run.Run();
+        }
+    }
+}

ContinuousTests/Tests/MarketplaceTest.cs

@@ -0,0 +1,25 @@
+using DistTestCore.Codex;
+
+namespace ContinuousTests.Tests
+{
+    //public class MarketplaceTest : ContinuousTest
+    //{
+    //    public override int RequiredNumberOfNodes => 1;
+    //    public override TimeSpan RunTestEvery => TimeSpan.FromDays(1);
+    //    public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+    //    [TestMoment(t: Zero)]
+    //    public void NodePostsStorageRequest()
+    //    {
+    //        //var c = new KubernetesWorkflow.WorkflowCreator(Log, new KubernetesWorkflow.Configuration());
+    //        //var flow = c.CreateWorkflow();
+    //        //var rc = flow.Start(10, KubernetesWorkflow.Location.Unspecified, new CodexContainerRecipe(), new KubernetesWorkflow.StartupConfig());
+    //    }
+
+    //    [TestMoment(t: DayThree)]
+    //    public void NodeDownloadsStorageRequestData()
+    //    {
+
+    //    }
+    //}
+}

ContinuousTests/Tests/PerformanceTests.cs

@@ -0,0 +1,86 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class UploadPerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+
+        [TestMoment(t: Zero)]
+        public void UploadTest()
+        {
+            UploadTest(100, Nodes[0]);
+        }
+    }
+
+    public class DownloadLocalPerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
+        {
+            DownloadTest(100, Nodes[0], Nodes[0]);
+        }
+    }
+
+    public class DownloadRemotePerformanceTest : PerformanceTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+
+        [TestMoment(t: Zero)]
+        public void DownloadTest()
+        {
+            DownloadTest(100, Nodes[0], Nodes[1]);
+        }
+    }
+
+    public abstract class PerformanceTest : ContinuousTest
+    {
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+        public void UploadTest(int megabytes, CodexNode uploadNode)
+        {
+            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+            var time = Measure(() =>
+            {
+                UploadFile(uploadNode, file);
+            });
+
+            var timePerMB = time / megabytes;
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
+        }
+
+        public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode)
+        {
+            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+            var cid = UploadFile(uploadNode, file);
+            Assert.That(cid, Is.Not.Null);
+
+            TestFile? result = null;
+            var time = Measure(() =>
+            {
+                result = DownloadContent(downloadNode, cid!);
+            });
+
+            file.AssertIsEqual(result);
+
+            var timePerMB = time / megabytes;
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+        }
+
+        private static TimeSpan Measure(Action action)
+        {
+            var start = DateTime.UtcNow;
+            action();
+            return DateTime.UtcNow - start;
+        }
+    }
+}

ContinuousTests/Tests/TwoClientTest.cs

@@ -0,0 +1,32 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class TwoClientTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            file = FileManager.GenerateTestFile(10.MB());
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+        }
+
+        [TestMoment(t: MinuteFive)]
+        public void DownloadTestFile()
+        {
+            var dl = DownloadContent(Nodes[1], cid!);
+
+            file.AssertIsEqual(dl);
+        }
+    }
+}

DistTestCore/Codex/CodexAccess.cs

@@ -10,68 +10,19 @@ namespace DistTestCore.Codex
         {
             this.lifecycle = lifecycle;
             Container = runningContainer;
+
+            var address = lifecycle.Configuration.GetAddress(Container);
+            Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address);
         }
 
         public RunningContainer Container { get; }
-
+        public CodexNode Node { get; }
-        public CodexDebugResponse GetDebugInfo()
-        {
-            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
-        {
-            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
-        {
-            var http = Http(timeout);
-            var str = http.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
-            {
-                return new CodexDebugPeerResponse
-                {
-                    IsPeerFound = false
-                };
-            }
-
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
-            result.IsPeerFound = true;
-            return result;
-        }
-
-        public string UploadFile(FileStream fileStream)
-        {
-            return Http().HttpPostStream("upload", fileStream);
-        }
-
-        public Stream DownloadFile(string contentId)
-        {
-            return Http().HttpGetStream("download/" + contentId);
-        }
-
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
-        {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
-        }
-
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
-        {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-        }
 
         public void EnsureOnline()
         {
             try
             {
-                var debugInfo = GetDebugInfo();
+                var debugInfo = Node.GetDebugInfo();
                 if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
 
                 var nodePeerId = debugInfo.id;
@@ -85,106 +36,5 @@ namespace DistTestCore.Codex
                 throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
             }
         }
-
-        private Http Http(TimeSpan? timeoutOverride = null)
-        {
-            var address = lifecycle.Configuration.GetAddress(Container);
-            return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride);
-        }
-    }
-
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set;}
     }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -50,7 +50,7 @@ namespace DistTestCore.Codex
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
                 var companionNode = gethConfig.CompanionNode;
-                var companionNodeAccount = companionNode.Accounts[Index];
+                var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)];
                 Additional(companionNodeAccount);
 
                 var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
@@ -60,7 +60,18 @@ namespace DistTestCore.Codex
                 AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                 AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
                 AddEnvVar("PERSISTENCE", "1");
+
+                if (config.MarketplaceConfig.IsValidator)
+                {
+                    AddEnvVar("VALIDATOR", "1");
+                }
             }
         }
+
+        private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
+        {
+            if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
+            return Index;
+        }
     }
 }

DistTestCore/Codex/CodexDeployment.cs

@@ -0,0 +1,45 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexDeployment
+    {
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata)
+        {
+            GethStartResult = gethStartResult;
+            CodexContainers = codexContainers;
+            Metadata = metadata;
+        }
+
+        public GethStartResult GethStartResult { get; }
+        public RunningContainer[] CodexContainers { get; }
+        public DeploymentMetadata Metadata { get; }
+    }
+
+    public class DeploymentMetadata
+    {
+        public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel)
+        {
+            DeployDateTimeUtc = DateTime.UtcNow;
+            CodexImage = codexImage;
+            GethImage = gethImage;
+            ContractsImage = contractsImage;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuotaMB = storageQuotaMB;
+            CodexLogLevel = codexLogLevel;
+        }
+        
+        public string CodexImage { get; }
+        public DateTime DeployDateTimeUtc { get; }
+        public string GethImage { get; }
+        public string ContractsImage { get; }
+        public string KubeNamespace { get; }
+        public int NumberOfCodexNodes { get; }
+        public int NumberOfValidators { get; }
+        public int StorageQuotaMB { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+    }
+}

DistTestCore/Codex/CodexNode.cs

@@ -0,0 +1,173 @@
+using Logging;
+using Utils;
+
+namespace DistTestCore.Codex
+{
+    public class CodexNode
+    {
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+
+        public CodexNode(BaseLog log, ITimeSet timeSet, Address address)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            Address = address;
+        }
+
+        public Address Address { get; }
+
+        public CodexDebugResponse GetDebugInfo()
+        {
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            var http = Http(timeout);
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
+            {
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
+            }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
+        }
+
+        public string UploadFile(FileStream fileStream)
+        {
+            return Http().HttpPostStream("upload", fileStream);
+        }
+
+        public Stream DownloadFile(string contentId)
+        {
+            return Http().HttpGetStream("download/" + contentId);
+        }
+
+        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        {
+            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+        }
+
+        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        {
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
+        }
+
+        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        {
+            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+        }
+
+        private Http Http(TimeSpan? timeoutOverride = null)
+        {
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
+        }
+    }
+
+    public class CodexDebugResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string[] addrs { get; set; } = new string[0];
+        public string repo { get; set; } = string.Empty;
+        public string spr { get; set; } = string.Empty;
+        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
+        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
+        public CodexDebugVersionResponse codex { get; set; } = new();
+        public CodexDebugTableResponse table { get; set; } = new();
+    }
+
+    public class CodexDebugTableResponse
+    {
+        public CodexDebugTableNodeResponse localNode { get; set; } = new();
+        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
+    }
+
+    public class CodexDebugTableNodeResponse
+    {
+        public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
+        public string record { get; set; } = string.Empty;
+        public string address { get; set; } = string.Empty;
+        public bool seen { get; set; }
+    }
+
+    public class EnginePeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public EnginePeerContextResponse context { get; set; } = new();
+    }
+
+    public class EnginePeerContextResponse
+    {
+        public int blocks { get; set; } = 0;
+        public int peerWants { get; set; } = 0;
+        public int exchanged { get; set; } = 0;
+        public string lastExchange { get; set; } = string.Empty;
+    }
+
+    public class SwitchPeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public string key { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugVersionResponse
+    {
+        public string version { get; set; } = string.Empty;
+        public string revision { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugPeerResponse
+    {
+        public bool IsPeerFound { get; set; }
+
+        public string peerId { get; set; } = string.Empty;
+        public long seqNo { get; set; }
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesAvailabilityRequest
+    {
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesAvailabilityResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesRequestStorageRequest
+    {
+        public string duration { get; set; } = string.Empty;
+        public string proofProbability { get; set; } = string.Empty;
+        public string reward { get; set; } = string.Empty;
+        public string collateral { get; set; } = string.Empty;
+        public string? expiry { get; set; }
+        public uint? nodes { get; set; }
+        public uint? tolerance { get; set; }
+    }
+}

DistTestCore/CodexSetup.cs

@@ -13,6 +13,7 @@ namespace DistTestCore
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
     }
     
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -62,7 +63,12 @@ namespace DistTestCore
 
         public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
         {
-            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance);
+            return EnableMarketplace(initialBalance, initialEther, false);
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
+        {
+            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator);
             return this;
         }
 

DistTestCore/Configuration.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore
 {
@@ -16,10 +17,20 @@ namespace DistTestCore
         {
             kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
             logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
-            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
-            codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+        }
+
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
+        {
+            this.kubeConfigFile = kubeConfigFile;
+            this.logPath = logPath;
+            this.logDebug = logDebug;
+            this.dataFilesPath = dataFilesPath;
+            this.codexLogLevel = codexLogLevel;
+            this.runnerLocation = runnerLocation;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
@@ -52,7 +63,7 @@ namespace DistTestCore
             return runnerLocation;
         }
 
-        public RunningContainerAddress GetAddress(RunningContainer container)
+        public Address GetAddress(RunningContainer container)
         {
             if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
             {
@@ -74,11 +85,6 @@ namespace DistTestCore
             if (v == null) return defaultValue;
             return v;
         }
-
-        private static T ParseEnum<T>(string value)
-        {
-            return (T)Enum.Parse(typeof(T), value, true);
-        }
     }
 
     public enum TestRunnerLocation

DistTestCore/FileManager.cs

@@ -19,11 +19,11 @@ namespace DistTestCore
         public const int ChunkSize = 1024 * 1024 * 100;
         private static NumberSource folderNumberSource = new NumberSource(0);
         private readonly Random random = new Random();
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly string folder;
         private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
 
-        public FileManager(TestLog log, Configuration configuration)
+        public FileManager(BaseLog log, Configuration configuration)
         {
             folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
 
@@ -142,9 +142,9 @@ namespace DistTestCore
 
     public class TestFile
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
 
-        public TestFile(TestLog log, string filename, string label)
+        public TestFile(BaseLog log, string filename, string label)
         {
             this.log = log;
             Filename = filename;

DistTestCore/GethStarter.cs

@@ -33,6 +33,8 @@ namespace DistTestCore
 
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
+            if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
+
             var interaction = marketplaceNetwork.StartInteraction(lifecycle);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 

DistTestCore/Http.cs

@@ -1,5 +1,4 @@
-using KubernetesWorkflow;
+using Logging;
-using Logging;
 using Newtonsoft.Json;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
@@ -11,11 +10,11 @@ namespace DistTestCore
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
-        private readonly RunningContainerAddress address;
+        private readonly Address address;
         private readonly string baseUrl;
         private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null)
         {
             this.log = log;
             this.timeSet = timeSet;

DistTestCore/Marketplace/GethStartResult.cs

@@ -1,4 +1,6 @@
-namespace DistTestCore.Marketplace
+using Newtonsoft.Json;
+
+namespace DistTestCore.Marketplace
 {
     public class GethStartResult
     {
@@ -9,6 +11,7 @@
             CompanionNode = companionNode;
         }
 
+        [JsonIgnore]
         public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
         public MarketplaceNetwork MarketplaceNetwork { get; }
         public GethCompanionNodeInfo CompanionNode { get; }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
                 $"proofProbability: {proofProbability}, " +
                 $"duration: {Time.FormatDuration(duration)})");
 
-            var response = codexAccess.RequestStorage(request, contentId.Id);
+            var response = codexAccess.Node.RequestStorage(request, contentId.Id);
 
             if (response == "Purchasing not available")
             {
@@ -78,7 +78,7 @@ namespace DistTestCore.Marketplace
                 $"maxCollateral: {maxCollateral}, " +
                 $"maxDuration: {Time.FormatDuration(maxDuration)})");
 
-            var response = codexAccess.SalesAvailability(request);
+            var response = codexAccess.Node.SalesAvailability(request);
 
             Log($"Storage successfully made available. Id: {response.id}");
 

DistTestCore/Marketplace/MarketplaceInitialConfig.cs

@@ -2,13 +2,16 @@
 {
     public class MarketplaceInitialConfig
     {
-        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens)
+        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
         {
             InitialEth = initialEth;
             InitialTestTokens = initialTestTokens;
+            IsValidator = isValidator;
         }
 
         public Ether InitialEth { get; }
         public TestToken InitialTestTokens { get; }
+        public bool IsValidator { get; }
+        public int? AccountIndexOverride { get; set; }
     }
 }

DistTestCore/OnlineCodexNode.cs

@@ -49,7 +49,7 @@ namespace DistTestCore
 
         public CodexDebugResponse GetDebugInfo()
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = CodexAccess.Node.GetDebugInfo();
             var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
             Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
             return debugInfo;
@@ -57,12 +57,12 @@ namespace DistTestCore
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId)
         {
-            return CodexAccess.GetDebugPeer(peerId);
+            return CodexAccess.Node.GetDebugPeer(peerId);
         }
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
         {
-            return CodexAccess.GetDebugPeer(peerId, timeout);
+            return CodexAccess.Node.GetDebugPeer(peerId, timeout);
         }
 
         public ContentId UploadFile(TestFile file)
@@ -72,7 +72,7 @@ namespace DistTestCore
             var logMessage = $"Uploading file {file.Describe()}...";
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream);
+                return CodexAccess.Node.UploadFile(fileStream);
             });
 
             if (response.StartsWith(UploadFailedMessage))
@@ -101,7 +101,7 @@ namespace DistTestCore
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
 
             Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
             Log($"Successfully connected to peer {peer.GetName()}.");
@@ -141,7 +141,7 @@ namespace DistTestCore
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                using var downloadStream = CodexAccess.Node.DownloadFile(contentId);
                 downloadStream.CopyTo(fileStream);
             }
             catch

DistTestCore/TestLifecycle.cs

@@ -7,15 +7,18 @@ namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private readonly WorkflowCreator workflowCreator;
         private DateTime testStart = DateTime.MinValue;
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
+            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
+        {
+        }
+
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
-            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
 
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this, workflowCreator);

KubernetesWorkflow/RunningContainers.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Utils;
+
+namespace KubernetesWorkflow
 {
     public class RunningContainers
     {
@@ -21,12 +23,12 @@
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
-            Name = GetContainerName(recipe, startupConfig);
+            Name = name;
             ClusterExternalAddress = clusterExternalAddress;
             ClusterInternalAddress = clusterInternalAddress;
         }
@@ -35,31 +37,7 @@
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
-        public RunningContainerAddress ClusterExternalAddress { get; }
+        public Address ClusterExternalAddress { get; }
-        public RunningContainerAddress ClusterInternalAddress { get; }
+        public Address ClusterInternalAddress { get; }
-
-        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
-        {
-            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
-            {
-                return $"<{startupConfig.NameOverride}{recipe.Number}>";
-            }
-            else
-            {
-                return $"<{recipe.Name}>";
-            }
-        }
-    }
-
-    public class RunningContainerAddress
-    {
-        public RunningContainerAddress(string host, int port)
-        {
-            Host = host;
-            Port = port;
-        }
-
-        public string Host { get; }
-        public int Port { get; }
     }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -80,27 +81,42 @@ namespace KubernetesWorkflow
                 var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
                 log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, startupConfig,
+                var name = GetContainerName(r, startupConfig);
+
+                return new RunningContainer(runningPod, r, servicePorts, name,
                     GetContainerExternalAddress(runningPod, servicePorts),
                     GetContainerInternalAddress(r));
 
             }).ToArray();
         }
 
-        private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
         {
-            return new RunningContainerAddress(
+            if (startupConfig == null) return "";
+            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
+            {
+                return $"<{startupConfig.NameOverride}{recipe.Number}>";
+            }
+            else
+            {
+                return $"<{recipe.Name}>";
+            }
+        }
+
+        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        {
+            return new Address(
                 pod.Cluster.HostAddress,
                 GetServicePort(servicePorts));
         }
 
-        private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe)
+        private Address GetContainerInternalAddress(ContainerRecipe recipe)
         {
             var serviceName = "service-" + numberSource.WorkflowNumber;
             var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
             var port = GetInternalPort(recipe);
 
-            return new RunningContainerAddress(
+            return new Address(
                 $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                 port);
         }

KubernetesWorkflow/WorkflowCreator.cs

@@ -13,10 +13,15 @@ namespace KubernetesWorkflow
         private readonly string testNamespace;
 
         public WorkflowCreator(BaseLog log, Configuration configuration)
+            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
+        {
+        }
+
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
+            testNamespace = testNamespacePostfix;
         }
 
         public StartupWorkflow CreateWorkflow()

Logging/BaseLog.cs

@@ -25,12 +25,12 @@ namespace Logging
             }
         }
 
-        public void Log(string message)
+        public virtual void Log(string message)
         {
             LogFile.Write(ApplyReplacements(message));
         }
 
-        public void Debug(string message = "", int skipFrames = 0)
+        public virtual void Debug(string message = "", int skipFrames = 0)
         {
             if (debug)
             {
@@ -40,24 +40,29 @@ namespace Logging
             }
         }
 
-        public void Error(string message)
+        public virtual void Error(string message)
         {
             Log($"[ERROR] {message}");
         }
 
-        public void MarkAsFailed()
+        public virtual void MarkAsFailed()
         {
             if (hasFailed) return;
             hasFailed = true;
             LogFile.ConcatToFilename("_FAILED");
         }
 
-        public void AddStringReplace(string from, string to)
+        public virtual void AddStringReplace(string from, string to)
         {
             if (string.IsNullOrWhiteSpace(from)) return;
             replacements.Add(new BaseLogStringReplacement(from, to));
         }
 
+        public virtual void Delete()
+        {
+            File.Delete(LogFile.FullFilename);
+        }
+
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)

Logging/FixtureLog.cs

@@ -8,19 +8,24 @@ namespace Logging
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config)
+        public FixtureLog(LogConfig config, string name = "")
             : base(config.DebugEnabled)
         {
             start = DateTime.UtcNow;
             var folder = DetermineFolder(config);
-            var fixtureName = GetFixtureName();
+            var fixtureName = GetFixtureName(name);
             fullName = Path.Combine(folder, fixtureName);
             this.config = config;
         }
 
-        public TestLog CreateTestLog()
+        public TestLog CreateTestLog(string name = "")
         {
-            return new TestLog(fullName, config.DebugEnabled);
+            return new TestLog(fullName, config.DebugEnabled, name);
+        }
+
+        public void DeleteFolder()
+        {
+            Directory.Delete(fullName, true);
         }
 
         protected override LogFile CreateLogFile()
@@ -36,10 +41,12 @@ namespace Logging
                Pad(start.Day));
         }
 
-        private string GetFixtureName()
+        private string GetFixtureName(string name)
         {
             var test = TestContext.CurrentContext.Test;
             var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            if (!string.IsNullOrEmpty(name)) className = name;
+
             return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
         }
 

Logging/TestLog.cs

@@ -9,10 +9,10 @@ namespace Logging
         private readonly string methodName;
         private readonly string fullName;
 
-        public TestLog(string folder, bool debug)
+        public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = GetMethodName();
+            methodName = GetMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -39,13 +39,15 @@ namespace Logging
                 MarkAsFailed();
             }
         }
+
         protected override LogFile CreateLogFile()
         {
             return new LogFile(fullName, "log");
         }
 
-        private string GetMethodName()
+        private string GetMethodName(string name)
         {
+            if (!string.IsNullOrEmpty(name)) return name;
             var test = TestContext.CurrentContext.Test;
             var args = FormatArguments(test);
             return $"{test.MethodName}{args}";

Utils/Address.cs

@@ -0,0 +1,14 @@
+namespace Utils
+{
+    public class Address
+    {
+        public Address(string host, int port)
+        {
+            Host = host;
+            Port = port;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+    }
+}

Utils/ParseEnum.cs

@@ -0,0 +1,10 @@
+namespace Utils
+{
+    public static class ParseEnum
+    {
+        public static T Parse<T>(string value)
+        {
+            return (T)Enum.Parse(typeof(T), value, true);
+        }
+    }
+}

Utils/RandomUtils.cs

@@ -0,0 +1,15 @@
+namespace Utils
+{
+    public static class RandomUtils
+    {
+        private static readonly Random random = new Random();
+
+        public static T PickOneRandom<T>(this List<T> remainingItems)
+        {
+            var i = random.Next(0, remainingItems.Count);
+            var result = remainingItems[i];
+            remainingItems.RemoveAt(i);
+            return result;
+        }
+    }
+}

cs-codex-dist-testing.sln

@@ -17,6 +17,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -51,6 +55,14 @@ Global
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE