Merge branch 'feature/continuous-test-future-counting' into feature/codex-net-deployer

# Conflicts:
#	CodexNetDeployer/Configuration.cs

ArgsUniform/ArgsUniform.cs

@@ -4,6 +4,7 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
+        private readonly Action printAppInfo;
         private readonly object? defaultsProvider;
         private readonly IEnv.IEnv env;
         private readonly string[] args;
@@ -12,23 +13,24 @@ namespace ArgsUniform
         private const int envStart = 48;
         private const int descStart = 80;
 
-        public ArgsUniform(params string[] args)
-            : this(new IEnv.Env(), args)
+        public ArgsUniform(Action printAppInfo, params string[] args)
+            : this(printAppInfo, new IEnv.Env(), args)
         {
         }
 
-        public ArgsUniform(object defaultsProvider, params string[] args)
-            : this(defaultsProvider, new IEnv.Env(), args)
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
+            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
         {
         }
 
-        public ArgsUniform(IEnv.IEnv env, params string[] args)
-            : this(null!, env, args)
+        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
+            : this(printAppInfo, null!, env, args)
         {
         }
 
-        public ArgsUniform(object defaultsProvider, IEnv.IEnv env, params string[] args)
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
+            this.printAppInfo = printAppInfo;
             this.defaultsProvider = defaultsProvider;
             this.env = env;
             this.args = args;
@@ -36,6 +38,13 @@ namespace ArgsUniform
 
         public T Parse(bool printResult = false)
         {
+            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+            {
+                printAppInfo();
+                PrintHelp();
+                throw new Exception();
+            }
+
             var result = Activator.CreateInstance<T>();
             var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
             var missingRequired = new List<PropertyInfo>();
@@ -201,14 +210,6 @@ namespace ArgsUniform
             }
         }
 
-        private bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString()!.ToLowerInvariant();
-            var isTrue = (s == "1" || s == "true");
-            uniformProperty.SetValue(result, isTrue);
-            return true;
-        }
-
         private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
         {
             var s = value.ToString();
@@ -230,6 +231,15 @@ namespace ArgsUniform
             return false;
         }
 
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (value != null)
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
         private string? GetFromArgs(string key)
         {
             var argKey = $"--{key}=";

ArgsUniform/ExampleUser.cs

@@ -21,9 +21,14 @@
             // env var: "AAA=BBB"
             var args = "--ccc=ddd";
 
-            var uniform = new ArgsUniform<Args>(new DefaultsProvider(), args);
+            var uniform = new ArgsUniform<Args>(PrintHelp, new DefaultsProvider(), args);
 
             var aaa = uniform.Parse();
         }
+
+        private static void PrintHelp()
+        {
+            Console.WriteLine("Help text!");
+        }
     }
 }

CodexNetDeployer/CodexNodeStarter.cs

@@ -34,38 +34,46 @@ namespace CodexNetDeployer
             var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
 
             var container = containers.Containers.First();
-            var codexAccess = new CodexAccess(lifecycle, container);
-
+            var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
             var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
             var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
             var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
 
-            var debugInfo = codexAccess.Node.GetDebugInfo();
-            if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+            try
             {
-                Console.Write("Online\t");
-
-                var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
-                interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
-                Console.Write("Tokens minted\t");
-
-                var response = marketAccess.MakeStorageAvailable(
-                    totalSpace: (config.StorageQuota!.Value - 1).MB(),
-                    minPriceForTotalSpace: config.MinPrice.TestTokens(),
-                    maxCollateral: config.MaxCollateral.TestTokens(),
-                    maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
-
-                if (!string.IsNullOrEmpty(response))
+                var debugInfo = codexAccess.GetDebugInfo();
+                if (!string.IsNullOrWhiteSpace(debugInfo.spr))
                 {
-                    Console.Write("Storage available\tOK" + Environment.NewLine);
+                    Console.Write("Online\t");
 
-                    if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
-                    validatorsLeft--;
-                    return container;
+                    var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
+                    Console.Write("Tokens minted\t");
+
+                    var response = marketAccess.MakeStorageAvailable(
+                        totalSpace: config.StorageSell!.Value.MB(),
+                        minPriceForTotalSpace: config.MinPrice.TestTokens(),
+                        maxCollateral: config.MaxCollateral.TestTokens(),
+                        maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
+
+                    if (!string.IsNullOrEmpty(response))
+                    {
+                        Console.Write("Storage available\tOK" + Environment.NewLine);
+
+                        if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+                        validatorsLeft--;
+                        return container;
+                    }
                 }
             }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Exception:" + ex.ToString());
+            }
+
+            Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
+            lifecycle.DownloadLog(container);
 
-            Console.Write("Unknown failure." + Environment.NewLine);
             return null;
         }
 
@@ -78,6 +86,10 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
+            if (config.BlockTTL != Configuration.SecondsIn1Day)
+            {
+                codexStart.BlockTTL = config.BlockTTL;
+            }
 
             return codexStart;
         }

CodexNetDeployer/Configuration.cs

@@ -7,6 +7,8 @@ namespace CodexNetDeployer
 {
     public class Configuration
     {
+        public const int SecondsIn1Day = 24 * 60 * 60;
+
         [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
         public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
 
@@ -28,9 +30,12 @@ namespace CodexNetDeployer
         [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
         public int? NumberOfValidators { get; set; }
 
-        [Uniform("storage-quota", "s", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
+        [Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
         public int? StorageQuota { get; set; }
 
+        [Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")]
+        public int? StorageSell { get; set; }
+
         [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
         public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
 
@@ -48,6 +53,9 @@ namespace CodexNetDeployer
 
         [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
         public bool RecordMetrics { get; set; } = false;
+       
+        [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
+        public int BlockTTL { get; set; } = SecondsIn1Day;
 
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
 
@@ -63,6 +71,10 @@ namespace CodexNetDeployer
             {
                 errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
             }
+            if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value)
+            {
+                errors.Add("StorageSell cannot be greater than or equal to StorageQuota.");
+            }
 
             return errors;
         }

CodexNetDeployer/Deployer.cs

@@ -36,6 +36,11 @@ namespace CodexNetDeployer
             Log("Geth started. Codex contracts deployed.");
             Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
 
+            // It takes a second for the geth node to unlock a single account. Let's assume 3.
+            // We can't start the codex nodes until their accounts are definitely unlocked. So
+            // We wait:
+            Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
+
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.

CodexNetDeployer/Program.cs

@@ -11,13 +11,7 @@ public class Program
         var nl = Environment.NewLine;
         Console.WriteLine("CodexNetDeployer" + nl);
 
-        if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
-        {
-            PrintHelp();
-            return;
-        }
-
-        var uniformArgs = new ArgsUniform<Configuration>(args);
+        var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
         if (args.Any(a => a == "--external"))
@@ -61,8 +55,5 @@ public class Program
 
         Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
             "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
-
-        var uniformArgs = new ArgsUniform<Configuration>();
-        uniformArgs.PrintHelp();
     }
 }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -0,0 +1,12 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --kube-namespace=codex-continuous-tests \
+    --nodes=5 \
+    --validators=3 \
+    --storage-quota=2048 \
+    --storage-sell=1024 \
+    --min-price=1024 \
+    --max-collateral=1024 \
+    --max-duration=3600000 \
+    --block-ttl=120
+

CodexNetDownloader/CodexNetDownloader.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDownloader/Configuration.cs

@@ -0,0 +1,22 @@
+using ArgsUniform;
+using DistTestCore;
+using DistTestCore.Codex;
+
+namespace CodexNetDownloader
+{
+    public class Configuration
+    {
+        [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
+        public string OutputPath { get; set; } = "output";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+    }
+}

CodexNetDownloader/Program.cs

@@ -0,0 +1,53 @@
+using ArgsUniform;
+using ContinuousTests;
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+using Newtonsoft.Json;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader" + nl);
+
+        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
+        var config = uniformArgs.Parse(true);
+
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
+        config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
+
+        if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
+
+        var k8sFactory = new K8sFactory();
+        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+
+        foreach (var container in config.CodexDeployment.CodexContainers)
+        {
+            lifecycle.DownloadLog(container);
+        }
+
+        Console.WriteLine("Done!");
+    }
+
+    private static CodexDeployment ParseCodexDeploymentJson(string filename)
+    {
+        var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+        if (d == null) throw new Exception("Unable to parse " + filename);
+        return d;
+    }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+
+        Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+    }
+}

ContinuousTests/CodexAccessFactory.cs

@@ -0,0 +1,20 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class CodexAccessFactory
+    {
+        public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+        {
+            return containers.Select(container =>
+            {
+                var address = container.ClusterExternalAddress;
+                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                return new CodexAccess(log, container, timeSet, address);
+            }).ToArray();
+        }
+    }
+}

ContinuousTests/CodexNodeFactory.cs

@@ -1,19 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using Logging;
-
-namespace ContinuousTests
-{
-    public class CodexNodeFactory
-    {
-        public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
-        {
-            return containers.Select(container =>
-            {
-                var address = container.ClusterExternalAddress;
-                return new CodexNode(log, timeSet, address);
-            }).ToArray();
-        }
-    }
-}

ContinuousTests/Configuration.cs

@@ -1,4 +1,5 @@
 using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
 using Newtonsoft.Json;
 
@@ -21,18 +22,27 @@ namespace ContinuousTests
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
+        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
+        public bool StopOnFailure { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 
     public class ConfigLoader
     {
         public Configuration Load(string[] args)
         {
-            var uniformArgs = new ArgsUniform<Configuration>(args);
+            var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
 
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
+            if (args.Any(a => a == "--external"))
+            {
+                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+            }
 
             return result;
         }
@@ -43,5 +53,14 @@ namespace ContinuousTests
             if (d == null) throw new Exception("Unable to parse " + filename);
             return d;
         }
+
+        private static void PrintHelp()
+        {
+            var nl = Environment.NewLine;
+            Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+
+            Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+        }
     }
 }

ContinuousTests/ContinuousTest.cs

@@ -21,23 +21,37 @@ namespace ContinuousTests
 
         private const string UploadFailedMessage = "Unable to store block";
 
-        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration)
+        public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
         {
             Nodes = nodes;
             Log = log;
             FileManager = fileManager;
             Configuration = configuration;
+            CancelToken = cancelToken;
+
+            if (nodes != null)
+            {
+                NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
+            }
+            else
+            {
+                NodeRunner = null!;
+            }
         }
 
-        public CodexNode[] Nodes { get; private set; } = null!;
+        public CodexAccess[] Nodes { get; private set; } = null!;
         public BaseLog Log { get; private set; } = null!;
         public IFileManager FileManager { get; private set; } = null!;
         public Configuration Configuration { get; private set; } = null!;
         public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+        public CancellationToken CancelToken { get; private set; } = new CancellationToken();
+        public NodeRunner NodeRunner { get; private set; } = null!;
 
         public abstract int RequiredNumberOfNodes { get; }
         public abstract TimeSpan RunTestEvery { get; }
         public abstract TestFailMode TestFailMode { get; }
+        public virtual int EthereumAccountIndex { get { return -1; } }
+        public virtual string CustomK8sNamespace { get { return string.Empty; } }
 
         public string Name
         {
@@ -47,7 +61,7 @@ namespace ContinuousTests
             }
         }
 
-        public ContentId? UploadFile(CodexNode node, TestFile file)
+        public ContentId? UploadFile(CodexAccess node, TestFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
 
@@ -57,15 +71,14 @@ namespace ContinuousTests
                 return node.UploadFile(fileStream);
             });
 
-            if (response.StartsWith(UploadFailedMessage))
-            {
-                return null;
-            }
+            if (string.IsNullOrEmpty(response)) return null;
+            if (response.StartsWith(UploadFailedMessage)) return null;
+            
             Log.Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
-        public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
+        public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             var file = FileManager.CreateEmptyTestFile(fileLabel);
@@ -74,7 +87,7 @@ namespace ContinuousTests
             return file;
         }
 
-        private void DownloadToFile(CodexNode node, string contentId, TestFile file)
+        private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);
             try

ContinuousTests/ContinuousTestRunner.cs

@@ -1,38 +1,65 @@
-using Logging;
+using DistTestCore;
+using Logging;
 
 namespace ContinuousTests
 {
     public class ContinuousTestRunner
     {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
         private readonly ConfigLoader configLoader = new ConfigLoader();
         private readonly TestFactory testFactory = new TestFactory();
         private readonly Configuration config;
         private readonly StartupChecker startupChecker;
+        private readonly CancellationToken cancelToken;
 
-        public ContinuousTestRunner(string[] args)
+        public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
         {
             config = configLoader.Load(args);
-            startupChecker = new StartupChecker(config);
+            startupChecker = new StartupChecker(config, cancelToken);
+            this.cancelToken = cancelToken;
         }
 
         public void Run()
         {
             startupChecker.Check();
 
+            var taskFactory = new TaskFactory();
             var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
-            var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
 
-            foreach (var t in testLoop)
+            ClearAllCustomNamespaces(allTests, overviewLog);
+
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
+
+            foreach (var testLoop in testLoops)
             {
-                overviewLog.Log("Launching test-loop for " + t.Name);
-                t.Begin();
-                Thread.Sleep(TimeSpan.FromMinutes(5));
+                if (cancelToken.IsCancellationRequested) break;
+
+                overviewLog.Log("Launching test-loop for " + testLoop.Name);
+                testLoop.Begin();
+                Thread.Sleep(TimeSpan.FromSeconds(5));
             }
 
-            overviewLog.Log("All test-loops launched.");
-            while (true) Thread.Sleep((2 ^ 31) - 1);
+            overviewLog.Log("Finished launching test-loops.");
+            cancelToken.WaitHandle.WaitOne();
+            overviewLog.Log("Cancelling all test-loops...");
+            taskFactory.WaitAll();
+            overviewLog.Log("All tasks cancelled.");
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
+        {
+            foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
+        {
+            if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
+
+            log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
+            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
+            workflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -0,0 +1,41 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class K8sFactory
+    {
+        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
+        {
+            var kubeConfig = GetKubeConfig(kubeConfigFile);
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: logPath,
+                logDebug: false,
+                dataFilesPath: dataFilePath,
+                codexLogLevel: CodexLogLevel.Debug,
+                runnerLocation: runnerLocation
+            );
+
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: customNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+            retryDelay: timeSet.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
+
+            return (workflowCreator, lifecycle);
+        }
+
+        private static string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+    }
+}

ContinuousTests/NodeRunner.cs

@@ -0,0 +1,97 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+using Logging;
+using Utils;
+
+namespace ContinuousTests
+{
+    public class NodeRunner
+    {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
+        private readonly CodexAccess[] nodes;
+        private readonly Configuration config;
+        private readonly ITimeSet timeSet;
+        private readonly BaseLog log;
+        private readonly string customNamespace;
+        private readonly int ethereumAccountIndex;
+
+        public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
+        {
+            this.nodes = nodes;
+            this.config = config;
+            this.timeSet = timeSet;
+            this.log = log;
+            this.customNamespace = customNamespace;
+            this.ethereumAccountIndex = ethereumAccountIndex;
+        }
+
+        public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(bootstrapNode, operation, 0.TestTokens());
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
+        {
+            var (workflowCreator, lifecycle) = CreateFacilities();
+            var flow = workflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = bootstrapNode.GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                startupConfig.NameOverride = "TransientNode";
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
+                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
+                codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                startupConfig.Add(config.CodexDeployment.GethStartResult);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+
+                var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
+
+                var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
+                if (mintTestTokens.Amount > 0)
+                {
+                    var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
+                    var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
+                }
+
+                var container = rc.Containers[0];
+                var address = lifecycle.Configuration.GetAddress(container);
+                var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
+                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
+
+                try
+                {
+                    operation(codexAccess, marketAccess, lifecycle);
+                }
+                catch
+                {
+                    lifecycle.DownloadLog(container);
+                    throw;
+                }
+            }
+            finally
+            {
+                flow.DeleteTestResources();
+            }
+        }
+
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        {
+            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
+        }
+    }
+}

ContinuousTests/Program.cs

@@ -6,7 +6,28 @@ public class Program
     {
         Console.WriteLine("Codex Continous-Test-Runner.");
         Console.WriteLine("Running...");
-        var runner = new ContinuousTestRunner(args);
+
+        var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token);
+
+        Console.CancelKeyPress += (sender, e) =>
+        {
+            Console.WriteLine("Stopping...");
+            e.Cancel = true;
+
+            Cancellation.Cts.Cancel();
+        };
+        
         runner.Run();
+        Console.WriteLine("Done.");
+    }
+
+    public static class Cancellation
+    {
+        static Cancellation()
+        {
+            Cts = new CancellationTokenSource();
+        }
+
+        public static CancellationTokenSource Cts { get; } 
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -5,45 +5,50 @@ using Utils;
 using KubernetesWorkflow;
 using NUnit.Framework.Internal;
 using System.Reflection;
+using static Program;
 
 namespace ContinuousTests
 {
     public class SingleTestRun
     {
-        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
         private readonly List<Exception> exceptions = new List<Exception>();
+        private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly TestHandle handle;
-        private readonly CodexNode[] nodes;
+        private readonly CancellationToken cancelToken;
+        private readonly CodexAccess[] nodes;
         private readonly FileManager fileManager;
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
 
-        public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
         {
+            this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.handle = handle;
-
+            this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
 
             nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
             fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
 
-        public void Run()
+        public void Run(EventWaitHandle runFinishedHandle)
         {
-            Task.Run(() =>
+            taskFactory.Run(() =>
             {
                 try
                 {
                     RunTest();
                     fileManager.DeleteAllTestFiles();
                     Directory.Delete(dataFolder, true);
+                    runFinishedHandle.Set();
                 }
                 catch (Exception ex)
                 {
@@ -65,6 +70,14 @@ namespace ContinuousTests
             {
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
+
+                if (config.StopOnFailure)
+                {
+                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
+                    DownloadClusterLogs();
+                    OverviewLog("Log download finished. Cancelling test runner...");
+                    Cancellation.Cts.Cancel();
+                }
             }
         }
 
@@ -75,6 +88,8 @@ namespace ContinuousTests
             var t = earliestMoment;
             while (true)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 RunMoment(t);
 
                 if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
@@ -86,9 +101,10 @@ namespace ContinuousTests
                 var nextMoment = handle.GetNextMoment(t);
                 if (nextMoment != null)
                 {
-                    Log($" > Next TestMoment in {nextMoment.Value} seconds...");
-                    t += nextMoment.Value;
-                    Thread.Sleep(nextMoment.Value * 1000);
+                    var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
+                    Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
+                    cancelToken.WaitHandle.WaitOne(delta);
+                    t = nextMoment.Value;
                 }
                 else
                 {
@@ -96,7 +112,7 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
-                    OverviewLog(" > Test passed.");
+                    OverviewLog(" > Test passed. " + FuturesInfo());
                     return;
                 }
             }
@@ -106,10 +122,30 @@ namespace ContinuousTests
         {
             var ex = UnpackException(exceptions.First());
             Log(ex.ToString());
-            OverviewLog(" > Test failed: " + ex.Message);
+            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
             throw ex;
         }
 
+        private string FuturesInfo()
+        {
+            var containers = config.CodexDeployment.CodexContainers;
+            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
+            var f = nodes.Select(n => n.GetDebugFutures().ToString());
+            var msg = $"(Futures: [{string.Join(", ", f)}])";
+            return msg;
+        }
+
+        private void DownloadClusterLogs()
+        {
+            var k8sFactory = new K8sFactory();
+            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+
+            foreach (var container in config.CodexDeployment.CodexContainers)
+            {
+                lifecycle.DownloadLog(container);
+            }
+        }
+
         private Exception UnpackException(Exception exception)
         {
             if (exception is AggregateException a)
@@ -144,12 +180,12 @@ namespace ContinuousTests
         private void InitializeTest(string name)
         {
             Log($" > Running TestMoment '{name}'");
-            handle.Test.Initialize(nodes, fixtureLog, fileManager, config);
+            handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
         }
 
         private void DecommissionTest()
         {
-            handle.Test.Initialize(null!, null!, null!, null!);
+            handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
         }
 
         private void Log(string msg)
@@ -160,14 +196,15 @@ namespace ContinuousTests
         private void OverviewLog(string msg)
         {
             Log(msg);
-            overviewLog.Log(testName + ": " +  msg);
+            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+            overviewLog.Log($"{containerNames} {testName}: {msg}");
         }
 
-        private CodexNode[] CreateRandomNodes(int number)
+        private CodexAccess[] CreateRandomNodes(int number)
         {
             var containers = SelectRandomContainers(number);
             fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
-            return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet);
+            return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
         }
 
         private RunningContainer[] SelectRandomContainers(int number)
@@ -184,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster);
+                CodexLogLevel.Error, config.RunnerLocation);
         }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -7,12 +7,14 @@ namespace ContinuousTests
     public class StartupChecker
     {
         private readonly TestFactory testFactory = new TestFactory();
-        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
         private readonly Configuration config;
+        private readonly CancellationToken cancelToken;
 
-        public StartupChecker(Configuration config)
+        public StartupChecker(Configuration config, CancellationToken cancelToken)
         {
             this.config = config;
+            this.cancelToken = cancelToken;
         }
 
         public void Check()
@@ -35,25 +37,19 @@ namespace ContinuousTests
             }
             foreach (var test in tests)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 var handle = new TestHandle(test);
                 handle.GetEarliestMoment();
                 handle.GetLastMoment();
             }
 
-            var errors = new List<string>();
-            foreach (var test in tests)
-            {
-                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
-                {
-                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
-                }
-            }
-
             if (!Directory.Exists(config.LogPath))
             {
                 Directory.CreateDirectory(config.LogPath);
             }
 
+            var errors = CheckTests(tests);
             if (errors.Any())
             {
                 throw new Exception("Prerun check failed: " + string.Join(", ", errors));
@@ -62,10 +58,12 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+            var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
             var pass = true;
             foreach (var n in nodes)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 log.Log($"Checking '{n.Address.Host}'...");
 
                 if (EnsureOnline(n))
@@ -84,7 +82,7 @@ namespace ContinuousTests
             }
         }
 
-        private bool EnsureOnline(CodexNode n)
+        private bool EnsureOnline(CodexAccess n)
         {
             try
             {
@@ -98,5 +96,69 @@ namespace ContinuousTests
             return true;
         }
 
+        private List<string> CheckTests(ContinuousTest[] tests)
+        {
+            var errors = new List<string>();
+            CheckRequiredNumberOfNodes(tests, errors);
+            CheckCustomNamespaceClashes(tests, errors);
+            CheckEthereumIndexClashes(tests, errors);
+            return errors;
+        }
+
+        private void CheckEthereumIndexClashes(ContinuousTest[] tests, List<string> errors)
+        {
+            var offLimits = config.CodexDeployment.CodexContainers.Length;
+            foreach (var test in tests)
+            {
+                if (test.EthereumAccountIndex != -1)
+                {
+                    if (test.EthereumAccountIndex <= offLimits)
+                    {
+                        errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'.");
+                    }
+                }
+            }
+
+            DuplicatesCheck(tests, errors,
+                considerCondition: t => t.EthereumAccountIndex != -1,
+                getValue: t => t.EthereumAccountIndex,
+                propertyName: nameof(ContinuousTest.EthereumAccountIndex));
+        }
+
+        private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors)
+        {
+            DuplicatesCheck(tests, errors,
+                considerCondition: t => !string.IsNullOrEmpty(t.CustomK8sNamespace),
+                getValue: t => t.CustomK8sNamespace,
+                propertyName: nameof(ContinuousTest.CustomK8sNamespace));
+        }
+
+        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
+        {
+            foreach (var test in tests)
+            {
+                if (considerCondition(test))
+                {
+                    var duplicates = tests.Where(t => t != test && getValue(t) == getValue(test)).ToList();
+                    if (duplicates.Any())
+                    {
+                        duplicates.Add(test);
+                        errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
+                        return;
+                    }
+                }
+            }
+        }
+
+        private void CheckRequiredNumberOfNodes(ContinuousTest[] tests, List<string> errors)
+        {
+            foreach (var test in tests)
+            {
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                {
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                }
+            }
+        }
     }
 }

ContinuousTests/TaskFactory.cs

@@ -0,0 +1,39 @@
+namespace ContinuousTests
+{
+    public class TaskFactory
+    {
+        private readonly object taskLock = new();
+        private readonly List<Task> activeTasks = new List<Task>();
+
+        public void Run(Action action)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        public void WaitAll()
+        {
+            var tasks = activeTasks.ToArray();
+            Task.WaitAll(tasks);
+
+            var moreTasks = false;
+            lock (taskLock)
+            {
+                activeTasks.RemoveAll(task => task.IsCompleted);
+                moreTasks = activeTasks.Any();
+            }
+
+            if (moreTasks) WaitAll();
+        }
+
+        private void CleanupTask(Task completedTask, object? arg)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Remove(completedTask);
+            }
+        }
+    }
+}

ContinuousTests/TestLoop.cs

@@ -4,18 +4,22 @@ namespace ContinuousTests
 {
     public class TestLoop
     {
+        private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
+        private readonly CancellationToken cancelToken;
+        private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
 
-        public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
         {
+            this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
-
+            this.cancelToken = cancelToken;
             Name = testType.Name;
         }
 
@@ -23,17 +27,26 @@ namespace ContinuousTests
 
         public void Begin()
         {
-            Task.Run(() =>
+            taskFactory.Run(() =>
             {
                 try
                 {
                     while (true)
                     {
+                        WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle });
+
+                        cancelToken.ThrowIfCancellationRequested();
+
                         StartTest();
-                        Thread.Sleep(runsEvery);
+
+                        cancelToken.WaitHandle.WaitOne(runsEvery);
                     }
                 }
-                catch(Exception ex)
+                catch (OperationCanceledException)
+                {
+                    overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
+                }
+                catch (Exception ex)
                 {
                     overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
                     Environment.Exit(-1);
@@ -45,8 +58,10 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(config, overviewLog, handle);
-            run.Run();
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
+
+            runFinishedHandle.Reset();
+            run.Run(runFinishedHandle);
         }
     }
 }

ContinuousTests/Tests/MarketplaceTest.cs

@@ -1,150 +1,99 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-using Logging;
-using Newtonsoft.Json;
-using NUnit.Framework;
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using Newtonsoft.Json;
+//using NUnit.Framework;
+//using Utils;
 
-namespace ContinuousTests.Tests
-{
-    public class MarketplaceTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => 1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromDays(4);
-        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+//namespace ContinuousTests.Tests
+//{
+//    public class MarketplaceTest : ContinuousTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
+//        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+//        public override int EthereumAccountIndex => 200;
+//        public override string CustomK8sNamespace => "codex-continuous-marketplace";
 
-        public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
-        public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
+//        private readonly uint numberOfSlots = 3;
+//        private readonly ByteSize fileSize = 10.MB();
+//        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
 
-        private readonly uint numberOfSlots = 3;
-        private readonly ByteSize fileSize = 10.MB();
-        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
+//        private TestFile file = null!;
+//        private ContentId? cid;
+//        private string purchaseId = string.Empty;
 
-        private TestFile file = null!;
-        private ContentId? cid;
-        private string purchaseId = string.Empty;
+//        [TestMoment(t: Zero)]
+//        public void NodePostsStorageRequest()
+//        {
+//            var contractDuration = TimeSpan.FromMinutes(8);
+//            decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
+//            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000;
 
-        [TestMoment(t: Zero)]
-        public void NodePostsStorageRequest()
-        {
-            var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
-            decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
-            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
-            Log.Log("expected total cost: " + expectedTotalCost);
+//            file = FileManager.GenerateTestFile(fileSize);
 
-            file = FileManager.GenerateTestFile(fileSize);
+//            NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
+//            {
+//                cid = UploadFile(codexAccess.Node, file);
+//                Assert.That(cid, Is.Not.Null);
 
-            var (workflowCreator, lifecycle) = CreateFacilities();
-            var flow = workflowCreator.CreateWorkflow();
+//                purchaseId = marketplaceAccess.RequestStorage(
+//                    contentId: cid!,
+//                    pricePerSlotPerSecond: pricePerSlotPerSecond,
+//                    requiredCollateral: 100.TestTokens(),
+//                    minRequiredNumberOfNodes: numberOfSlots,
+//                    proofProbability: 10,
+//                    duration: contractDuration);
 
-            try
-            {
-                var debugInfo = Nodes[0].GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+//                Assert.That(!string.IsNullOrEmpty(purchaseId));
 
-                var startupConfig = new StartupConfig();
-                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
-                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
-                codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
-                codexStartConfig.BootstrapSpr = debugInfo.spr;
-                startupConfig.Add(codexStartConfig);
-                startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
-                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+//                WaitForContractToStart(codexAccess, purchaseId);
+//            });
+//        }
 
-                var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
-                var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
+//        [TestMoment(t: MinuteFive + MinuteOne)]
+//        public void StoredDataIsAvailableAfterThreeDays()
+//        {
+//            NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
+//            {
+//                var result = DownloadFile(codexAccess.Node, cid!);
 
-                var interaction = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
-                interaction.MintTestTokens(new[] { account.Account }, expectedTotalCost, tokenAddress);
+//                file.AssertIsEqual(result);
+//            });
+//        }
 
-                var container = rc.Containers[0];
-                var marketplaceNetwork = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork;
-                var codexAccess = new CodexAccess(lifecycle, container);
-                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
+//        private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId)
+//        {
+//            var lastState = "";
+//            var waitStart = DateTime.UtcNow;
+//            var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024);
+//            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
 
-                cid = UploadFile(codexAccess.Node, file);
-                Assert.That(cid, Is.Not.Null);
+//            Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}");
+//            while (lastState != "started")
+//            {
+//                CancelToken.ThrowIfCancellationRequested();
 
-                var balance = marketAccess.GetBalance();
-                Log.Log("Account: " + account.Account);
-                Log.Log("Balance: " + balance);
+//                var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
+//                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+//                if (purchaseStatus != null && purchaseStatus.state != lastState)
+//                {
+//                    lastState = purchaseStatus.state;
+//                    Log.Log("Purchase status: " + statusJson);
+//                }
 
-                purchaseId = marketAccess.RequestStorage(
-                    contentId: cid!,
-                    pricePerSlotPerSecond: pricePerSlotPerSecond,
-                    requiredCollateral: 100.TestTokens(),
-                    minRequiredNumberOfNodes: numberOfSlots,
-                    proofProbability: 10,
-                    duration: contractDuration);
+//                Thread.Sleep(2000);
 
-                Log.Log($"PurchaseId: '{purchaseId}'");
-                Assert.That(!string.IsNullOrEmpty(purchaseId));
-            }
-            finally
-            {
-                flow.DeleteTestResources();
-            }
-        }
+//                if (lastState == "errored")
+//                {
+//                    Assert.Fail("Contract start failed: " + statusJson);
+//                }
 
-        [TestMoment(t: DayThree)]
-        public void StoredDataIsAvailableAfterThreeDays()
-        {
-            var (workflowCreator, lifecycle) = CreateFacilities();
-            var flow = workflowCreator.CreateWorkflow();
-          
-            try
-            {
-                var debugInfo = Nodes[0].GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
-
-                var startupConfig = new StartupConfig();
-                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
-                codexStartConfig.BootstrapSpr = debugInfo.spr;
-                startupConfig.Add(codexStartConfig);
-                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
-                var container = rc.Containers[0];
-                var codexAccess = new CodexAccess(lifecycle, container);
-
-                var result = DownloadContent(codexAccess.Node, cid!);
-
-                file.AssertIsEqual(result);
-            }
-            finally
-            {
-                flow.DeleteTestResources();
-            }
-        }
-
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
-        {
-            var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
-            var lifecycleConfig = new DistTestCore.Configuration
-            (
-                kubeConfigFile: kubeConfig,
-                logPath: "null",
-                logDebug: false,
-                dataFilesPath: Configuration.LogPath,
-                codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: TestRunnerLocation.ExternalToCluster
-            );
-
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: MarketplaceTestNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: TimeSet.K8sOperationTimeout(),
-            retryDelay: TimeSet.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
-
-            return (workflowCreator, lifecycle);
-        }
-
-        private string? GetKubeConfig(string kubeConfigFile)
-        {
-            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
-            return kubeConfigFile;
-        }
-    }
-}
+//                if (DateTime.UtcNow - waitStart > maxWaitTime)
+//                {
+//                    Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}");
+//                }
+//            }
+//            Log.Log("Contract started.");
+//        }
+//    }
+//}

ContinuousTests/Tests/PerformanceTests.cs

@@ -1,86 +1,86 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using NUnit.Framework;
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using NUnit.Framework;
 
-namespace ContinuousTests.Tests
-{
-    public class UploadPerformanceTest : PerformanceTest
-    {
-        public override int RequiredNumberOfNodes => 1;
+//namespace ContinuousTests.Tests
+//{
+//    public class UploadPerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
 
-        [TestMoment(t: Zero)]
-        public void UploadTest()
-        {
-            UploadTest(100, Nodes[0]);
-        }
-    }
+//        [TestMoment(t: Zero)]
+//        public void UploadTest()
+//        {
+//            UploadTest(100, Nodes[0]);
+//        }
+//    }
 
-    public class DownloadLocalPerformanceTest : PerformanceTest
-    {
-        public override int RequiredNumberOfNodes => 1;
+//    public class DownloadLocalPerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
 
-        [TestMoment(t: Zero)]
-        public void DownloadTest()
-        {
-            DownloadTest(100, Nodes[0], Nodes[0]);
-        }
-    }
+//        [TestMoment(t: Zero)]
+//        public void DownloadTest()
+//        {
+//            DownloadTest(100, Nodes[0], Nodes[0]);
+//        }
+//    }
 
-    public class DownloadRemotePerformanceTest : PerformanceTest
-    {
-        public override int RequiredNumberOfNodes => 2;
+//    public class DownloadRemotePerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 2;
 
-        [TestMoment(t: Zero)]
-        public void DownloadTest()
-        {
-            DownloadTest(100, Nodes[0], Nodes[1]);
-        }
-    }
+//        [TestMoment(t: Zero)]
+//        public void DownloadTest()
+//        {
+//            DownloadTest(100, Nodes[0], Nodes[1]);
+//        }
+//    }
 
-    public abstract class PerformanceTest : ContinuousTest
-    {
-        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
-        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+//    public abstract class PerformanceTest : ContinuousTest
+//    {
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
+//        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
 
-        public void UploadTest(int megabytes, CodexNode uploadNode)
-        {
-            var file = FileManager.GenerateTestFile(megabytes.MB());
+//        public void UploadTest(int megabytes, CodexAccess uploadNode)
+//        {
+//            var file = FileManager.GenerateTestFile(megabytes.MB());
 
-            var time = Measure(() =>
-            {
-                UploadFile(uploadNode, file);
-            });
+//            var time = Measure(() =>
+//            {
+//                UploadFile(uploadNode, file);
+//            });
 
-            var timePerMB = time / megabytes;
+//            var timePerMB = time / megabytes;
 
-            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
-        }
+//            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
+//        }
 
-        public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode)
-        {
-            var file = FileManager.GenerateTestFile(megabytes.MB());
+//        public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode)
+//        {
+//            var file = FileManager.GenerateTestFile(megabytes.MB());
 
-            var cid = UploadFile(uploadNode, file);
-            Assert.That(cid, Is.Not.Null);
+//            var cid = UploadFile(uploadNode, file);
+//            Assert.That(cid, Is.Not.Null);
 
-            TestFile? result = null;
-            var time = Measure(() =>
-            {
-                result = DownloadContent(downloadNode, cid!);
-            });
+//            TestFile? result = null;
+//            var time = Measure(() =>
+//            {
+//                result = DownloadFile(downloadNode, cid!);
+//            });
 
-            file.AssertIsEqual(result);
+//            file.AssertIsEqual(result);
 
-            var timePerMB = time / megabytes;
+//            var timePerMB = time / megabytes;
 
-            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
-        }
+//            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+//        }
 
-        private static TimeSpan Measure(Action action)
-        {
-            var start = DateTime.UtcNow;
-            action();
-            return DateTime.UtcNow - start;
-        }
-    }
-}
+//        private static TimeSpan Measure(Action action)
+//        {
+//            var start = DateTime.UtcNow;
+//            action();
+//            return DateTime.UtcNow - start;
+//        }
+//    }
+//}

ContinuousTests/Tests/ThresholdChecks.cs

@@ -0,0 +1,60 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class ThresholdChecks : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private static readonly List<string> previousBreaches = new List<string>();
+
+        [TestMoment(t: 0)]
+        public void CheckAllThresholds()
+        {
+            var allNodes = CreateAccessToAllNodes();
+            foreach (var n in allNodes) CheckThresholds(n);
+        }
+
+        private void CheckThresholds(CodexAccess n)
+        {
+            var breaches = n.GetDebugThresholdBreaches();
+            if (breaches.breaches.Any())
+            {
+                var newBreaches = new List<string>();
+                foreach (var b in breaches.breaches)
+                {
+                    if (!previousBreaches.Contains(b))
+                    {
+                        newBreaches.Add(b);
+                        previousBreaches.Add(b);
+                    }
+                }
+
+                if (newBreaches.Any())
+                {
+                    Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
+
+                    Program.Cancellation.Cts.Cancel();
+                }
+            }
+        }
+
+        private string FormatBreach(CodexAccess n, string breach)
+        {
+            return $"{n.Container.Name} = '{breach}'";
+        }
+
+        private CodexAccess[] CreateAccessToAllNodes()
+        {
+            // Normally, a continuous test accesses only a subset of the nodes in the deployment.
+            // This time, we want to check all of them.
+            var factory = new CodexAccessFactory();
+            var allContainers = Configuration.CodexDeployment.CodexContainers;
+            return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
+        }
+    }
+}

ContinuousTests/Tests/TransientNodeTest.cs

@@ -0,0 +1,54 @@
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using NUnit.Framework;
+
+//namespace ContinuousTests.Tests
+//{
+//    public class TransientNodeTest : ContinuousTest
+//    {
+//        public override int RequiredNumberOfNodes => 3;
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
+//        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+//        public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant();
+//        public override int EthereumAccountIndex => 201;
+
+//        private TestFile file = null!;
+//        private ContentId cid = null!;
+
+//        private CodexAccess UploadBootstapNode { get { return Nodes[0]; } }
+//        private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } }
+//        private CodexAccess IntermediateNode { get { return Nodes[2]; } }
+
+//        [TestMoment(t: 0)]
+//        public void UploadWithTransientNode()
+//        {
+//            file = FileManager.GenerateTestFile(10.MB());
+
+//            NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
+//            {
+//                cid = UploadFile(codexAccess, file)!;
+//                Assert.That(cid, Is.Not.Null);
+
+//                var dlt = Task.Run(() =>
+//                {
+//                    Thread.Sleep(10000);
+//                    lifecycle.DownloadLog(codexAccess.Container);
+//                });
+
+//                var resultFile = DownloadFile(IntermediateNode, cid);
+//                dlt.Wait();
+//                file.AssertIsEqual(resultFile);
+//            });
+//        }
+
+//        [TestMoment(t: 30)]
+//        public void DownloadWithTransientNode()
+//        {
+//            NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
+//            {
+//                var resultFile = DownloadFile(codexAccess, cid);
+//                file.AssertIsEqual(resultFile);
+//            });
+//        }
+//    }
+//}

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;
@@ -21,10 +21,10 @@ namespace ContinuousTests.Tests
             Assert.That(cid, Is.Not.Null);
         }
 
-        [TestMoment(t: MinuteFive)]
+        [TestMoment(t: 10)]
         public void DownloadTestFile()
         {
-            var dl = DownloadContent(Nodes[1], cid!);
+            var dl = DownloadFile(Nodes[1], cid!);
 
             file.AssertIsEqual(dl);
         }

ContinuousTests/run-continuous-tests.sh

@@ -0,0 +1,4 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --codex-deployment=codex-deployment.json \
+    --stop=1 

DistTestCore/Codex/CodexAccess.cs

@@ -1,40 +1,96 @@
 using KubernetesWorkflow;
+using Logging;
+using Utils;
 
 namespace DistTestCore.Codex
 {
     public class CodexAccess
     {
-        private readonly TestLifecycle lifecycle;
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
 
-        public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
+        public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
-            this.lifecycle = lifecycle;
-            Container = runningContainer;
-
-            var address = lifecycle.Configuration.GetAddress(Container);
-            Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address);
+            this.log = log;
+            Container = container;
+            this.timeSet = timeSet;
+            Address = address;
         }
 
         public RunningContainer Container { get; }
-        public CodexNode Node { get; }
+        public Address Address { get; }
 
-        public void EnsureOnline()
+        public CodexDebugResponse GetDebugInfo()
         {
-            try
-            {
-                var debugInfo = Node.GetDebugInfo();
-                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
+        }
 
-                var nodePeerId = debugInfo.id;
-                var nodeName = Container.Name;
-                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            }
-            catch (Exception e)
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            var http = Http(timeout);
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
             {
-                lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
-                throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
             }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
+        }
+
+        public int GetDebugFutures()
+        {
+            return Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
+        }
+
+        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
+        {
+            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+        }
+
+        public string UploadFile(FileStream fileStream)
+        {
+            return Http().HttpPostStream("upload", fileStream);
+        }
+
+        public Stream DownloadFile(string contentId)
+        {
+            return Http().HttpGetStream("download/" + contentId);
+        }
+
+        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        {
+            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+        }
+
+        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        {
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
+        }
+
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
+        }
+
+        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        {
+            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+        }
+
+        private Http Http(TimeSpan? timeoutOverride = null)
+        {
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride);
         }
     }
 }

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,86 +1,9 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexNode
-    {
-        private readonly BaseLog log;
-        private readonly ITimeSet timeSet;
-
-        public CodexNode(BaseLog log, ITimeSet timeSet, Address address)
-        {
-            this.log = log;
-            this.timeSet = timeSet;
-            Address = address;
-        }
-
-        public Address Address { get; }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
-        {
-            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
-        {
-            var http = Http(timeout);
-            var str = http.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
-            {
-                return new CodexDebugPeerResponse
-                {
-                    IsPeerFound = false
-                };
-            }
-
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
-            result.IsPeerFound = true;
-            return result;
-        }
-
-        public string UploadFile(FileStream fileStream)
-        {
-            return Http().HttpPostStream("upload", fileStream);
-        }
-
-        public Stream DownloadFile(string contentId)
-        {
-            return Http().HttpGetStream("download/" + contentId);
-        }
-
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
-        {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
-        }
-
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
-        {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-        }
-
-        private Http Http(TimeSpan? timeoutOverride = null)
-        {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
-        }
-    }
-
     public class CodexDebugResponse
     {
         public string id { get; set; } = string.Empty;
@@ -93,6 +16,11 @@ namespace DistTestCore.Codex
         public CodexDebugTableResponse table { get; set; } = new();
     }
 
+    public class CodexDebugFutures
+    {
+        public int futures { get; set; }
+    }
+
     public class CodexDebugTableResponse
     {
         public CodexDebugTableNodeResponse localNode { get; set; } = new();
@@ -148,6 +76,11 @@ namespace DistTestCore.Codex
         public string address { get; set; } = string.Empty;
     }
 
+    public class CodexDebugThresholdBreaches
+    {
+        public string[] breaches { get; set; } = Array.Empty<string>();
+    }
+
     public class CodexSalesAvailabilityRequest
     {
         public string size { get; set; } = string.Empty;

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -40,6 +40,10 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
             }
+            if (config.BlockTTL != null)
+            {
+                AddEnvVar("BLOCK_TTL", config.BlockTTL.ToString()!);
+            }
             if (config.MetricsEnabled)
             {
                 AddEnvVar("METRICS_ADDR", "0.0.0.0");

DistTestCore/Codex/CodexStartupConfig.cs

@@ -17,5 +17,6 @@ namespace DistTestCore.Codex
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
+        public int? BlockTTL { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -64,12 +64,19 @@ namespace DistTestCore
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
+            foreach (var node in Nodes)
+            {
+                var debugInfo = node.CodexAccess.GetDebugInfo();
+                var nodePeerId = debugInfo.id;
+                var nodeName = node.CodexAccess.Container.Name;
+                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+            }
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(lifecycle, c);
+            var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
             return factory.CreateOnlineCodexNode(access, this);
         }
     }

DistTestCore/FileManager.cs

@@ -1,6 +1,5 @@
 using Logging;
 using NUnit.Framework;
-using System.Runtime.InteropServices;
 using Utils;
 
 namespace DistTestCore

DistTestCore/Http.cs

@@ -12,14 +12,16 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
+        private readonly string? logAlias;
         private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
             this.timeoutOverride = timeoutOverride;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -113,7 +115,14 @@ namespace DistTestCore
 
         private void Log(string url, string message)
         {
-            log.Debug($"({url}) = '{message}'", 3);
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
         }
 
         private T Retry<T>(Func<T> operation, string description)

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -34,10 +34,10 @@ namespace DistTestCore.Marketplace
         {
             var request = new CodexSalesRequestStorageRequest
             {
-                duration = ToHexBigInt(duration.TotalSeconds),
-                proofProbability = ToHexBigInt(proofProbability),
-                reward = ToHexBigInt(pricePerSlotPerSecond),
-                collateral = ToHexBigInt(requiredCollateral),
+                duration = ToDecInt(duration.TotalSeconds),
+                proofProbability = ToDecInt(proofProbability),
+                reward = ToDecInt(pricePerSlotPerSecond),
+                collateral = ToDecInt(requiredCollateral),
                 expiry = null,
                 nodes = minRequiredNumberOfNodes,
                 tolerance = null,
@@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
                 $"proofProbability: {proofProbability}, " +
                 $"duration: {Time.FormatDuration(duration)})");
 
-            var response = codexAccess.Node.RequestStorage(request, contentId.Id);
+            var response = codexAccess.RequestStorage(request, contentId.Id);
 
             if (response == "Purchasing not available")
             {
@@ -66,10 +66,10 @@ namespace DistTestCore.Marketplace
         {
             var request = new CodexSalesAvailabilityRequest
             {
-                size = ToHexBigInt(totalSpace.SizeInBytes),
-                duration = ToHexBigInt(maxDuration.TotalSeconds),
-                maxCollateral = ToHexBigInt(maxCollateral),
-                minPrice = ToHexBigInt(minPriceForTotalSpace)
+                size = ToDecInt(totalSpace.SizeInBytes),
+                duration = ToDecInt(maxDuration.TotalSeconds),
+                maxCollateral = ToDecInt(maxCollateral),
+                minPrice = ToDecInt(minPriceForTotalSpace)
             };
 
             Log($"Making storage available... (" +
@@ -78,22 +78,23 @@ namespace DistTestCore.Marketplace
                 $"maxCollateral: {maxCollateral}, " +
                 $"maxDuration: {Time.FormatDuration(maxDuration)})");
 
-            var response = codexAccess.Node.SalesAvailability(request);
+            var response = codexAccess.SalesAvailability(request);
 
             Log($"Storage successfully made available. Id: {response.id}");
 
             return response.id;
         }
 
-        private string ToHexBigInt(double d)
+        private string ToDecInt(double d)
         {
-            return "0x" + string.Format("{0:X}", Convert.ToInt64(d));
+            var i = new BigInteger(d);
+            return i.ToString("D");
         }
 
-        public string ToHexBigInt(TestToken t)
+        public string ToDecInt(TestToken t)
         {
-            var bigInt = new BigInteger(t.Amount);
-            return "0x" + bigInt.ToString("X");
+            var i = new BigInteger(t.Amount);
+            return i.ToString("D");
         }
 
         public void AssertThatBalance(IResolveConstraint constraint, string message = "")

DistTestCore/Metrics/MetricsAccess.cs

@@ -14,12 +14,12 @@ namespace DistTestCore.Metrics
 
     public class MetricsAccess : IMetricsAccess
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly ITimeSet timeSet;
         private readonly MetricsQuery query;
         private readonly RunningContainer node;
 
-        public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
+        public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
         {
             this.log = log;
             this.timeSet = timeSet;

DistTestCore/Metrics/MetricsDownloader.cs

@@ -5,9 +5,9 @@ namespace DistTestCore.Metrics
 {
     public class MetricsDownloader
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
 
-        public MetricsDownloader(TestLog log)
+        public MetricsDownloader(BaseLog log)
         {
             this.log = log;
         }

DistTestCore/OnlineCodexNode.cs

@@ -49,7 +49,7 @@ namespace DistTestCore
 
         public CodexDebugResponse GetDebugInfo()
         {
-            var debugInfo = CodexAccess.Node.GetDebugInfo();
+            var debugInfo = CodexAccess.GetDebugInfo();
             var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
             Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
             return debugInfo;
@@ -57,12 +57,12 @@ namespace DistTestCore
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId)
         {
-            return CodexAccess.Node.GetDebugPeer(peerId);
+            return CodexAccess.GetDebugPeer(peerId);
         }
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
         {
-            return CodexAccess.Node.GetDebugPeer(peerId, timeout);
+            return CodexAccess.GetDebugPeer(peerId, timeout);
         }
 
         public ContentId UploadFile(TestFile file)
@@ -72,13 +72,12 @@ namespace DistTestCore
             var logMessage = $"Uploading file {file.Describe()}...";
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
-                return CodexAccess.Node.UploadFile(fileStream);
+                return CodexAccess.UploadFile(fileStream);
             });
 
-            if (response.StartsWith(UploadFailedMessage))
-            {
-                Assert.Fail("Node failed to store block.");
-            }
+            if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
+            if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
+
             var logReplacement = $"(CID:{file.Describe()})";
             Log($"ContentId '{response}' is {logReplacement}");
             lifecycle.Log.AddStringReplace(response, logReplacement);
@@ -101,7 +100,7 @@ namespace DistTestCore
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
 
             Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
             Log($"Successfully connected to peer {peer.GetName()}.");
@@ -141,7 +140,7 @@ namespace DistTestCore
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
-                using var downloadStream = CodexAccess.Node.DownloadFile(contentId);
+                using var downloadStream = CodexAccess.DownloadFile(contentId);
                 downloadStream.CopyTo(fileStream);
             }
             catch

DistTestCore/TestLifecycle.cs

@@ -9,12 +9,12 @@ namespace DistTestCore
     {
         private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
             : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
         {
         }
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
         {
             Log = log;
             Configuration = configuration;
@@ -27,7 +27,7 @@ namespace DistTestCore
             testStart = DateTime.UtcNow;
         }
 
-        public TestLog Log { get; }
+        public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }

Logging/BaseLog.cs

@@ -4,6 +4,7 @@ namespace Logging
 {
     public abstract class BaseLog
     {
+        private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly bool debug;
         private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private bool hasFailed;
@@ -14,17 +15,21 @@ namespace Logging
             this.debug = debug;
         }
 
-        protected abstract LogFile CreateLogFile();
+        protected abstract string GetFullName();
 
-        protected LogFile LogFile 
+        public LogFile LogFile 
         {
             get
             {
-                if (logFile == null) logFile = CreateLogFile();
+                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
                 return logFile;
             }
         }
 
+        public virtual void EndTest()
+        {
+        }
+
         public virtual void Log(string message)
         {
             LogFile.Write(ApplyReplacements(message));
@@ -63,6 +68,11 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+        }
+
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)
@@ -71,6 +81,11 @@ namespace Logging
             }
             return str;
         }
+
+        private string GetSubfileNumber()
+        {
+            return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
+        }
     }
 
     public class BaseLogStringReplacement

Logging/FixtureLog.cs

@@ -28,9 +28,9 @@ namespace Logging
             Directory.Delete(fullName, true);
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return new LogFile(fullName, "log");
+            return fullName;
         }
 
         private string DetermineFolder(LogConfig config)

Logging/NullLog.cs

@@ -6,9 +6,9 @@
         {
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return null!;
+            return "NULL";
         }
 
         public override void Log(string message)

Logging/TestLog.cs

@@ -1,11 +1,9 @@
 using NUnit.Framework;
-using Utils;
 
 namespace Logging
 {
     public class TestLog : BaseLog
     {
-        private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly string methodName;
         private readonly string fullName;
 
@@ -18,12 +16,7 @@ namespace Logging
             Log($"*** Begin: {methodName}");
         }
 
-        public LogFile CreateSubfile(string ext = "log")
-        {
-            return new LogFile($"{fullName}_{GetSubfileNumber()}", ext);
-        }
-
-        public void EndTest()
+        public override void EndTest()
         {
             var result = TestContext.CurrentContext.Result;
 
@@ -40,9 +33,9 @@ namespace Logging
             }
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return new LogFile(fullName, "log");
+            return fullName;
         }
 
         private string GetMethodName(string name)
@@ -53,11 +46,6 @@ namespace Logging
             return $"{test.MethodName}{args}";
         }
 
-        private string GetSubfileNumber()
-        {
-            return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
-        }
-
         private static string FormatArguments(TestContext.TestAdapter test)
         {
             if (test.Arguments == null || !test.Arguments.Any()) return "";

cs-codex-dist-testing.sln

@@ -21,7 +21,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "Continuo
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -69,6 +71,10 @@ Global
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{634324B1-E359-42B4-A269-BDC429936B3C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE