Merge branch 'master' into feature/waku-plugin

# Conflicts:
#	Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
#	cs-codex-dist-testing.sln

.github/workflows/docker-keymaker.yml

@@ -1,27 +0,0 @@
-name: Docker - KeyMaker
-
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*.*.*'
-    paths:
-      - 'Tools/KeyMaker/**'
-      - 'Framework/**'
-      - 'ProjectPlugins/**'
-      - .github/workflows/docker-KeyMaker.yml
-      - .github/workflows/docker-reusable.yml
-  workflow_dispatch:
-
-
-jobs:
-  build-and-push:
-    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
-    with:
-      docker_file: Tools/KeyMaker/docker/Dockerfile
-      docker_repo: codexstorage/codex-keymaker
-    secrets: inherit
-

Framework/ArgsUniform/ArgsUniform.cs

@@ -4,8 +4,9 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
-        private readonly Assigner<T> assigner;
         private readonly Action printAppInfo;
+        private readonly object? defaultsProvider;
+        private readonly IEnv.IEnv env;
         private readonly string[] args;
         private const int cliStart = 8;
         private const int shortStart = 38;
@@ -30,9 +31,9 @@ namespace ArgsUniform
         public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
             this.printAppInfo = printAppInfo;
+            this.defaultsProvider = defaultsProvider;
+            this.env = env;
             this.args = args;
-
-            assigner = new Assigner<T>(env, args, defaultsProvider);
         }
 
         public T Parse(bool printResult = false)
@@ -41,7 +42,7 @@ namespace ArgsUniform
             {
                 printAppInfo();
                 PrintHelp();
-                Environment.Exit(0);
+                throw new Exception();
             }
 
             var result = Activator.CreateInstance<T>();
@@ -52,16 +53,18 @@ namespace ArgsUniform
                 var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
                 if (attr != null)
                 {
-                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
                     {
-                        missingRequired.Add(uniformProperty);
+                        {
+                            missingRequired.Add(uniformProperty);
+                        }
                     }
                 }
             }
 
             if (missingRequired.Any())
             {
-                PrintResults(printResult,result, uniformProperties);
+                PrintResults(result, uniformProperties);
                 Print("");
                 foreach (var missing in missingRequired)
                 {
@@ -72,39 +75,37 @@ namespace ArgsUniform
                 }
 
                 PrintHelp();
-                Environment.Exit(1);
+                throw new ArgumentException("Unable to assemble all required arguments");
             }
 
-            PrintResults(printResult, result, uniformProperties);
+            if (printResult)
+            {
+                PrintResults(result, uniformProperties);
+            }
 
             return result;
         }
 
+        private void PrintResults(T result, PropertyInfo[] uniformProperties)
+        {
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
         public void PrintHelp()
         {
             Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
-            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
-            foreach (var prop in props)
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
+            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
+            foreach (var attr in attrs)
             {
-                var a = prop.GetCustomAttribute<UniformAttribute>();
-                if (a != null)
-                {
-                    var optional = !a.Required ? " (optional)" : "";
-                    var def = assigner.DescribeDefaultFor(prop);
-                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
-                }
-            }
-            Print("");
-        }
-
-        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
-        {
-            if (!printResult) return;
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
+                var a = attr!;
+                var optional = !a.Required ? " *" : "";
+                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
             }
             Print("");
         }
@@ -114,7 +115,7 @@ namespace ArgsUniform
             Console.WriteLine(msg);
         }
 
-        private void PrintAligned(string cli, string s, string env, string desc, string def)
+        private void PrintAligned(string cli, string s, string env, string desc)
         {
             Console.CursorLeft = cliStart;
             Console.Write(cli);
@@ -123,8 +124,132 @@ namespace ArgsUniform
             Console.CursorLeft = envStart;
             Console.Write(env);
             Console.CursorLeft = descStart;
-            Console.Write(desc + " ");
-            Console.Write(def + Environment.NewLine);
+            Console.Write(desc + Environment.NewLine);
+        }
+
+        private object GetDefaultValue(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+
+        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var currentValue = uniformProperty.GetValue(result);
+            var isEmptryString = (currentValue as string) == string.Empty;
+            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
+
+            if (defaultsProvider == null) return false;
+
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return false;
+
+            var value = defaultProperty.GetValue(defaultsProvider);
+            if (value != null)
+            {
+                return Assign(result, uniformProperty, value);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    
+                    throw new NotSupportedException();
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
         }
     }
 }

Framework/ArgsUniform/Assigner.cs

@@ -1,186 +0,0 @@
-using System.Globalization;
-using System.Numerics;
-using System.Reflection;
-
-namespace ArgsUniform
-{
-    public class Assigner<T>
-    {
-        private readonly IEnv.IEnv env;
-        private readonly string[] args;
-        private readonly object? defaultsProvider;
-
-        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
-        {
-            this.env = env;
-            this.args = args;
-            this.defaultsProvider = defaultsProvider;
-        }
-
-        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        public string DescribeDefaultFor(PropertyInfo property)
-        {
-            var obj = Activator.CreateInstance<T>();
-            var defaultValue = GetDefaultValue(obj, property);
-            if (defaultValue == null) return "";
-            if (defaultValue is string str)
-            {
-                return "\"" + str + "\"";
-            }
-            return defaultValue.ToString() ?? string.Empty;
-        }
-
-        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
-        {
-            // Get value from object's static initializer if it's there.
-            var currentValue = uniformProperty.GetValue(result);
-            if (currentValue != null) return currentValue;
-
-            // Get value from defaults-provider object if it's there.
-            if (defaultsProvider == null) return null;
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return null;
-            return defaultProperty.GetValue(defaultsProvider);
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var defaultValue = GetDefaultValue(result, uniformProperty);
-            var isEmptryString = (defaultValue as string) == string.Empty;
-            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
-            {
-                return Assign(result, uniformProperty, defaultValue);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
-
-                    throw new NotSupportedException(
-                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
-                        $"for property '${uniformProperty.Name}'.");
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
-        {
-            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
-        }
-
-        private static object GetDefaultValueForType(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-    }
-}

Framework/Core/CoreInterface.cs

@@ -30,11 +30,11 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var msg = $"Downloading container log for '{container.Name}'";
-            entryPoint.Tools.GetLog().Log(msg);
-            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
+            var file = entryPoint.Tools.GetLog().CreateSubfile();
+            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
+            var logHandler = new LogDownloadHandler(container.Name, file);
             workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return new DownloadedLog(logHandler);
+            return logHandler.DownloadLog();
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/DownloadedLog.cs

@@ -1,11 +1,9 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
 namespace Core
 {
     public interface IDownloadedLog
     {
-        void IterateLines(Action<string> action);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
@@ -15,22 +13,9 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(WriteToFileLogHandler logHandler)
+        internal DownloadedLog(LogFile logFile)
         {
-            logFile = logHandler.LogFile;
-        }
-
-        public void IterateLines(Action<string> action)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                action(line);
-                line = streamReader.ReadLine();
-            }
+            this.logFile = logFile;
         }
 
         public string[] GetLinesContaining(string expectedString)

Framework/Core/EntryPoint.cs

@@ -38,14 +38,10 @@ namespace Core
             return new CoreInterface(this);
         }
 
-        /// <summary>
-        /// Deletes kubernetes and tracked file resources.
-        /// when `waitTillDone` is true, this function will block until resources are deleted.
-        /// </summary>
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
         {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
         }
 
         internal T GetPlugin<T>() where T : IProjectPlugin

Framework/Core/Http.cs

@@ -7,7 +7,6 @@ namespace Core
     {
         T OnClient<T>(Func<HttpClient, T> action);
         T OnClient<T>(Func<HttpClient, T> action, string description);
-        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
         IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
@@ -36,19 +35,13 @@ namespace Core
         }
 
         public T OnClient<T>(Func<HttpClient, T> action, string description)
-        {
-            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
-            return OnClient(action, retry);
-        }
-
-        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
         {
             var client = GetClient();
 
             return LockRetry(() =>
             {
                 return action(client);
-            }, retry);
+            }, description);
         }
 
         public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
@@ -61,11 +54,11 @@ namespace Core
             return DebugStack.GetCallerName(skipFrames: 2);
         }
 
-        private T LockRetry<T>(Func<T> operation, Retry retry)
+        private T LockRetry<T>(Func<T> operation, string description)
         {
             lock (httpLock)
             {
-                return retry.Run(operation);
+                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
             }
         }
 

Framework/Core/LogDownloadHandler.cs

@@ -0,0 +1,28 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    internal class LogDownloadHandler : LogHandler, ILogHandler
+    {
+        private readonly LogFile log;
+
+        internal LogDownloadHandler(string description, LogFile log)
+        {
+            this.log = log;
+
+            log.Write($"{description} -->> {log.FullFilename}");
+            log.WriteRaw(description);
+        }
+
+        internal IDownloadedLog DownloadLog()
+        {
+            return new DownloadedLog(log);
+        }
+
+        protected override void ProcessLine(string line)
+        {
+            log.WriteRaw(line);
+        }
+    }
+}

Framework/Core/PluginManager.cs

@@ -34,12 +34,12 @@
             return metadata;
         }
 
-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
         {
             foreach (var pair in pairs)
             {
                 pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
             }
         }
 

Framework/Core/PluginTools.cs

@@ -6,13 +6,7 @@ namespace Core
 {
     public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
     {
-        ITimeSet TimeSet { get; }
-
-        /// <summary>
-        /// Deletes kubernetes and tracked file resources.
-        /// when `waitTillDone` is true, this function will block until resources are deleted.
-        /// </summary>
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
     }
 
     public interface IWorkflowTool
@@ -39,6 +33,7 @@ namespace Core
 
     internal class PluginTools : IPluginTools
     {
+        private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
         private readonly LogPrefixer log;
@@ -47,12 +42,10 @@ namespace Core
         {
             this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            TimeSet = timeSet;
+            this.timeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
-        public ITimeSet TimeSet { get; }
-
         public void ApplyLogPrefix(string prefix)
         {
             log.Prefix = prefix;
@@ -60,7 +53,7 @@ namespace Core
 
         public IHttp CreateHttp(Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(onClientCreated, TimeSet);
+            return CreateHttp(onClientCreated, timeSet);
         }
 
         public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
@@ -70,7 +63,7 @@ namespace Core
 
         public IHttp CreateHttp()
         {
-            return new Http(log, TimeSet);
+            return new Http(log, timeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
@@ -78,9 +71,9 @@ namespace Core
             return workflowCreator.CreateWorkflow(namespaceOverride);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
         {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
             if (deleteTrackedFiles) fileManager.DeleteAllFiles();
         }
 

Framework/Core/TimeSet.cs

@@ -2,31 +2,10 @@
 {
     public interface ITimeSet
     {
-        /// <summary>
-        /// Timeout for a single HTTP call.
-        /// </summary>
         TimeSpan HttpCallTimeout();
-
-        /// <summary>
-        /// Maximum total time to attempt to make a successful HTTP call to a service.
-        /// When HTTP calls time out during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan HttpRetryTimeout();
-
-        /// <summary>
-        /// After a failed HTTP call, wait this long before trying again.
-        /// </summary>
+        int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
+        TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
     }
 
@@ -37,9 +16,9 @@
             return TimeSpan.FromMinutes(3);
         }
 
-        public TimeSpan HttpRetryTimeout()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromMinutes(10);
+            return 3;
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -47,7 +26,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan K8sOperationRetryDelay()
+        public TimeSpan WaitForK8sServiceDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -62,27 +41,27 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(30);
+            return TimeSpan.FromHours(2);
         }
 
-        public TimeSpan HttpRetryTimeout()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromHours(2.2);
+            return 1;
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(20);
+            return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan K8sOperationRetryDelay()
+        public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(30);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromHours(1);
+            return TimeSpan.FromMinutes(15);
         }
     }
 }

Framework/DiscordRewards/CheckConfig.cs

@@ -13,9 +13,9 @@ namespace DiscordRewards
     public enum CheckType
     {
         Uninitialized,
-        HostFilledSlot,
-        HostFinishedSlot,
-        ClientPostedContract,
-        ClientStartedContract,
+        FilledSlot,
+        FinishedSlot,
+        PostedContract,
+        StartedContract,
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -5,11 +5,6 @@
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
         public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
         public string[] EventsOverview { get; set; } = Array.Empty<string>();
-
-        public bool HasAny()
-        {
-            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
-        }
     }
 
     public class RewardUsersCommand
@@ -21,7 +16,7 @@
     public class MarketAverage
     {
         public int NumberOfFinished { get; set; }
-        public int TimeRangeSeconds { get; set; }
+        public TimeSpan TimeRange { get; set; }
         public float Price { get; set; }
         public float Size { get; set; }
         public float Duration { get; set; }

Framework/DiscordRewards/RewardRepo.cs

@@ -11,19 +11,19 @@ namespace DiscordRewards
             // Filled any slot
             new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
             {
-                Type = CheckType.HostFilledSlot
+                Type = CheckType.FilledSlot
             }),
 
             // Finished any slot
             new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
             {
-                Type = CheckType.HostFinishedSlot
+                Type = CheckType.FinishedSlot
             }),
 
             // Finished a sizable slot
             new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
             {
-                Type = CheckType.HostFinishedSlot,
+                Type = CheckType.FinishedSlot,
                 MinSlotSize = 10.MB(),
                 MinDuration = TimeSpan.FromMinutes(5.0),
             }),
@@ -31,19 +31,19 @@ namespace DiscordRewards
             // Posted any contract
             new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
             {
-                Type = CheckType.ClientPostedContract
+                Type = CheckType.PostedContract
             }),
 
             // Started any contract
             new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
             {
-                Type = CheckType.ClientStartedContract
+                Type = CheckType.StartedContract
             }),
 
             // Started a sizable contract
             new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
             {
-                Type = CheckType.ClientStartedContract,
+                Type = CheckType.StartedContract,
                 MinNumberOfHosts = 4,
                 MinSlotSize = 10.MB(),
                 MinDuration = TimeSpan.FromMinutes(5.0),

Framework/FileUtils/TrackedFile.cs

@@ -65,7 +65,7 @@ namespace FileUtils
 
                 if (readExpected == 0 && readActual == 0)
                 {
-                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
+                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
                     return;
                 }
 

Framework/KubernetesWorkflow/ByteSizeExtensions.cs

@@ -0,0 +1,41 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public static class ByteSizeExtensions
+    {
+        public static string ToSuffixNotation(this ByteSize b)
+        {
+            long x = 1024;
+            var map = new Dictionary<long, string>
+            {
+                { Pow(x, 4), "Ti" },
+                { Pow(x, 3), "Gi" },
+                { Pow(x, 2), "Mi" },
+                { (x), "Ki" },
+            };
+
+            var bytes = b.SizeInBytes;
+            foreach (var pair in map)
+            {
+                if (bytes > pair.Key)
+                {
+                    double bytesD = bytes;
+                    double divD = pair.Key;
+                    double numD = Math.Ceiling(bytesD / divD);
+                    var v = Convert.ToInt64(numD);
+                    return $"{v}{pair.Value}";
+                }
+            }
+
+            return $"{bytes}";
+        }
+
+        private static long Pow(long x, int v)
+        {
+            long result = 1;
+            for (var i = 0; i < v; i++) result *= x;
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,6 +11,7 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
+        private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
@@ -26,10 +27,11 @@ namespace KubernetesWorkflow
             cts = new CancellationTokenSource();
         }
 
-        public void Start()
+        public void Start(ILogHandler logHandler)
         {
             if (worker != null) throw new InvalidOperationException();
 
+            this.logHandler = logHandler;
             cts = new CancellationTokenSource();
             worker = Task.Run(Worker);
         }
@@ -48,9 +50,7 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            var result = HasContainerBeenRestarted(client);
-            if (result) DownloadCrashedContainerLogs(client);
-            return result;
+            return HasContainerBeenRestarted(client);
         }
 
         private void Worker()
@@ -83,16 +83,14 @@ namespace KubernetesWorkflow
         private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
-            if (result) log.Log("Pod crash detected for " + containerName);
-            return result;
+            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
         }
 
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
+            log.Log("Pod crash detected for " + containerName);
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
-            handler.Log(stream);
+            logHandler!.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -16,7 +16,6 @@ namespace KubernetesWorkflow
         {
             var config = GetConfig();
             UpdateHostAddress(config);
-            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
             return config;
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,11 +43,6 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
-        public void WaitUntilOnline(RunningContainer container)
-        {
-            WaitUntilDeploymentOnline(container);
-        }
-
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -64,14 +59,14 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
 
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
@@ -115,7 +110,7 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
+        public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
 
@@ -124,28 +119,25 @@ namespace KubernetesWorkflow
 
             foreach (var ns in namespaces)
             {
-                DeleteNamespace(ns, wait);
+                DeleteNamespace(ns);
             }
         }
 
-        public void DeleteNamespace(bool wait)
+        public void DeleteNamespace()
         {
             log.Debug();
             if (IsNamespaceOnline(K8sNamespace))
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
-
-                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
             }
         }
 
-        public void DeleteNamespace(string ns, bool wait)
+        public void DeleteNamespace(string ns)
         {
             log.Debug();
             if (IsNamespaceOnline(ns))
             {
                 client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
-                if (wait) WaitUntilNamespaceDeleted(ns);
             }
         }
 
@@ -380,6 +372,7 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
+            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -535,7 +528,7 @@ namespace KubernetesWorkflow
             }
             if (set.Memory.SizeInBytes != 0)
             {
-                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
+                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
             }
             return result;
         }
@@ -708,14 +701,14 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxTimeout: TimeSpan.FromMinutes(1),
+                maxRetries: 6,
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -871,45 +864,16 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
         }
 
-        private void WaitUntilNamespaceDeleted(string @namespace)
-        {
-            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
-        }
-
-        private void WaitUntilDeploymentOnline(RunningContainer container)
+        private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
             {
-                CheckForCrash(container);
-
-                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            }, nameof(WaitUntilDeploymentOnline));
-        }
-
-        private void CheckForCrash(RunningContainer container)
-        {
-            var deploymentName = container.Recipe.Name;
-            var podName = GetPodName(container);
-
-            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
-            if (podInfo == null) return;
-            if (podInfo.Status == null) return;
-            if (podInfo.Status.ContainerStatuses == null) return;
-
-            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
-            if (result)
-            {
-                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
-                log.Error(msg);
-
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
-
-                throw new Exception(msg);
-            }
+            });
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -919,7 +883,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            }, nameof(WaitUntilDeploymentOffline));
+            });
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -928,19 +892,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            }, nameof(WaitUntilPodsForDeploymentAreOffline));
+            });
         }
 
-        private void WaitUntil(Func<bool> predicate, string msg)
+        private void WaitUntil(Func<bool> predicate)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
             }
             finally
             {
-                sw.End(msg, 1);
+                sw.End("", 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningPod runningPod);
-        void OnContainersStopped(RunningPod runningPod);
+        void OnContainersStarted(RunningContainers runningContainers);
+        void OnContainersStopped(RunningContainers runningContainers);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningPod runningPod)
+        public void OnContainersStarted(RunningContainers runningContainers)
         {
         }
 
-        public void OnContainersStopped(RunningPod runningPod)
+        public void OnContainersStopped(RunningContainers runningContainers)
         {
         }
 

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,6 +1,4 @@
-using Logging;
-
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
@@ -22,25 +20,4 @@ namespace KubernetesWorkflow
 
         protected abstract void ProcessLine(string line);
     }
-
-    public class WriteToFileLogHandler : LogHandler, ILogHandler
-    {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
-        {
-            LogFile = sourceLog.CreateSubfile();
-
-            var msg = $"{description} -->> {LogFile.FullFilename}";
-            sourceLog.Log(msg);
-
-            LogFile.Write(msg);
-            LogFile.WriteRaw(description);
-        }
-
-        public LogFile LogFile { get; }
-
-        protected override void ProcessLine(string line)
-        {
-            LogFile.WriteRaw(line);
-        }
-    }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -73,6 +73,13 @@ namespace KubernetesWorkflow.Recipe
             return p;
         }
 
+        protected Port AddInternalPort(int number, string tag = "", PortProtocol protocol = PortProtocol.TCP)
+        {
+            var p = factory.CreateInternalPort(number, tag, protocol);
+            internalPorts.Add(p);
+            return p;
+        }
+
         protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
             AddEnvVar(name, AddExposedPort(tag, protocol));
@@ -105,7 +112,7 @@ namespace KubernetesWorkflow.Recipe
 
         protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
         {
-            var size = 10.MB().SizeInBytes.ToString();
+            var size = 10.MB().ToSuffixNotation();
             volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
         }
 
@@ -114,7 +121,7 @@ namespace KubernetesWorkflow.Recipe
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                resourceQuantity: volumeSize.SizeInBytes.ToString()));
+                resourceQuantity: volumeSize.ToSuffixNotation()));
         }
 
         protected void Additional(object userData)

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -16,7 +16,12 @@ namespace KubernetesWorkflow.Recipe
 
         public Port CreateInternalPort(string tag, PortProtocol protocol)
         {
-            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+            return CreateInternalPort(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateInternalPort(int number, string tag, PortProtocol protocol)
+        {
+            return new Port(number, tag, protocol);
         }
 
         public Port CreateExternalPort(int number, string tag, PortProtocol protocol)

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,5 +1,8 @@
-using KubernetesWorkflow.Recipe;
+using k8s;
+using k8s.Models;
+using KubernetesWorkflow.Recipe;
 using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,16 +9,16 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningPod pod);
+        PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningPod pod, bool waitTillStopped);
-        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        void Stop(RunningContainers containers, bool waitTillStopped);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace(bool wait);
-        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
+        void DeleteNamespace();
+        void DeleteNamespacesStartingWith(string namespacePrefix);
     }
 
     public class StartupWorkflow : IStartupWorkflow
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,36 +60,25 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return new FutureContainers(rc, this);
-            });
-        }
-
-        public void WaitUntilOnline(RunningPod rc)
-        {
-            K8s(controller =>
-            {
-                foreach (var c in rc.Containers)
-                {
-                    controller.WaitUntilOnline(c);
-                }
+                return rc;
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningPod pod)
+        public PodInfo GetPodInfo(RunningContainers containers)
         {
-            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -97,20 +86,20 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningPod runningPod, bool waitTillStopped)
+        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
+                controller.Stop(runningContainers.StartResult, waitTillStopped);
+                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+                controller.DownloadPodLog(container, logHandler, tailLines);
             });
         }
 
@@ -122,19 +111,19 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteNamespace(bool wait)
+        public void DeleteNamespace()
         {
             K8s(controller =>
             {
-                controller.DeleteNamespace(wait);
+                controller.DeleteNamespace();
             });
         }
 
-        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
+        public void DeleteNamespacesStartingWith(string namespacePrefix)
         {
             K8s(controller =>
             {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -1,20 +0,0 @@
-namespace KubernetesWorkflow.Types
-{
-    public class FutureContainers
-    {
-        private readonly RunningPod runningPod;
-        private readonly StartupWorkflow workflow;
-
-        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
-        {
-            this.runningPod = runningPod;
-            this.workflow = workflow;
-        }
-
-        public RunningPod WaitForOnline()
-        {
-            workflow.WaitUntilOnline(runningPod);
-            return runningPod;
-        }
-    }
-}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
         public ContainerAddress[] Addresses { get; }
 
         [JsonIgnore]
-        public RunningPod RunningPod { get; internal set; } = null!;
+        public RunningContainers RunningContainers { get; internal set; } = null!;
 
         public Address GetAddress(ILog log, string portTag)
         {

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -2,15 +2,15 @@
 
 namespace KubernetesWorkflow.Types
 {
-    public class RunningPod
+    public class RunningContainers
     {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
 
-            foreach (var c in containers) c.RunningPod = this;
+            foreach (var c in containers) c.RunningContainers = this;
         }
 
         public StartupConfig StartupConfig { get; }
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public string Name
         {
-            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
+            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
         }
 
         public string Describe()
@@ -31,7 +31,12 @@ namespace KubernetesWorkflow.Types
 
     public static class RunningContainersExtensions
     {
-        public static string Describe(this RunningPod[] runningContainers)
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
         {
             return string.Join(",", runningContainers.Select(c => c.Describe()));
         }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,14 +18,6 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
-        public BlockTimeEntry Get(ulong blockNumber)
-        {
-            bounds.Initialize();
-            var b = cache.Get(blockNumber);
-            if (b != null) return b;
-            return GetBlock(blockNumber);
-        }
-
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
@@ -46,7 +38,7 @@ namespace NethereumWorkflow.BlockUtils
 
         private ulong Log(Func<ulong> operation)
         {
-            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder));
             var result = operation();
             sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
 

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -117,17 +117,9 @@ namespace NethereumWorkflow
             }
 
             return new BlockInterval(
-                timeRange: timeRange,
                 from: fromBlock.Value,
                 to: toBlock.Value
             );
         }
-
-        public BlockTimeEntry GetBlockForNumber(ulong number)
-        {
-            var wrapper = new Web3Wrapper(web3, log);
-            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
-            return blockTimeFinder.Get(number);
-        }
     }
 }

Framework/Utils/BlockInterval.cs

@@ -2,7 +2,7 @@
 {
     public class BlockInterval
     {
-        public BlockInterval(TimeRange timeRange, ulong from, ulong to)
+        public BlockInterval(ulong from, ulong to)
         {
             if (from < to)
             {
@@ -14,13 +14,10 @@
                 From = to;
                 To = from;
             }
-            TimeRange = timeRange;
         }
 
         public ulong From { get; }
         public ulong To { get; }
-        public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
 
         public override string ToString()
         {

Framework/Utils/Formatter.cs

@@ -1,6 +1,4 @@
-using System.Globalization;
-
-namespace Utils
+namespace Utils
 {
     public static class Formatter
     {
@@ -12,7 +10,7 @@ namespace Utils
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
+            return digit.ToString() + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,7 +2,6 @@
 {
     public class NumberSource
     {
-        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -12,12 +11,8 @@
 
         public int GetNextNumber()
         {
-            var n = -1;
-            lock (@lock)
-            {
-                n = number;
-                number++;
-            }
+            var n = number;
+            number++;
             return n;
         }
     }

Framework/Utils/Retry.cs

@@ -1,131 +0,0 @@
-namespace Utils
-{
-    public class Retry
-    {
-        private readonly string description;
-        private readonly TimeSpan maxTimeout;
-        private readonly TimeSpan sleepAfterFail;
-        private readonly Action<Failure> onFail;
-
-        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
-        {
-            this.description = description;
-            this.maxTimeout = maxTimeout;
-            this.sleepAfterFail = sleepAfterFail;
-            this.onFail = onFail;
-        }
-
-        public void Run(Action task)
-        {
-            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
-            run.Run();
-        }
-
-        public T Run<T>(Func<T> task)
-        {
-            T? result = default;
-
-            var run = new RetryRun(description, () =>
-            {
-                result = task();
-            }, maxTimeout, sleepAfterFail, onFail);
-            run.Run();
-
-            return result!;
-        }
-
-        private class RetryRun
-        {
-            private readonly string description;
-            private readonly Action task;
-            private readonly TimeSpan maxTimeout;
-            private readonly TimeSpan sleepAfterFail;
-            private readonly Action<Failure> onFail;
-            private readonly DateTime start = DateTime.UtcNow;
-            private readonly List<Failure> failures = new List<Failure>();
-            private int tryNumber;
-            private DateTime tryStart;
-
-            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
-            {
-                this.description = description;
-                this.task = task;
-                this.maxTimeout = maxTimeout;
-                this.sleepAfterFail = sleepAfterFail;
-                this.onFail = onFail;
-
-                tryNumber = 0;
-                tryStart = DateTime.UtcNow;
-            }
-
-            public void Run()
-            {
-                while (true)
-                {
-                    CheckMaximums();
-
-                    tryNumber++;
-                    tryStart = DateTime.UtcNow;
-                    try
-                    {
-                        task();
-                        return;
-                    }
-                    catch (Exception ex)
-                    {
-                        var failure = CaptureFailure(ex);
-                        onFail(failure);
-                        Time.Sleep(sleepAfterFail);
-                    }
-                }
-            }
-
-            private Failure CaptureFailure(Exception ex)
-            {
-                var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
-                failures.Add(f);
-                return f;
-            }
-
-            private void CheckMaximums()
-            {
-                if (Duration() > maxTimeout) Fail();
-            }
-
-            private void Fail()
-            {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
-                        new AggregateException(failures.Select(f => f.Exception)));
-            }
-
-            private string GetFailureReport()
-            {
-                return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
-            }
-
-            private TimeSpan Duration()
-            {
-                return DateTime.UtcNow - start;
-            }
-        }
-    }
-
-    public class Failure
-    {
-        public Failure(Exception exception, TimeSpan duration, int tryNumber)
-        {
-            Exception = exception;
-            Duration = duration;
-            TryNumber = tryNumber;
-        }
-
-        public Exception Exception { get; }
-        public TimeSpan Duration { get; }
-        public int TryNumber { get; }
-
-        public string Describe()
-        {
-            return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
-        }
-    }
-}

Framework/Utils/Time.cs

@@ -18,12 +18,6 @@
             task.Wait();
         }
 
-        public static string FormatDuration(TimeSpan? d)
-        {
-            if (d == null) return "[NULL]";
-            return FormatDuration(d.Value);
-        }
-
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -63,70 +57,100 @@
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate, string msg)
+        public static void WaitUntil(Func<bool> predicate)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
         {
             var start = DateTime.UtcNow;
-            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                var duration = DateTime.UtcNow - start;
-                if (duration > timeout)
+                if (DateTime.UtcNow - start > timeout)
                 {
-                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
+                    throw new TimeoutException("Operation timed out.");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
-                tries++;
             }
         }
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, TimeSpan.FromSeconds(30), description);
+            Retry(action, 1, description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, TimeSpan.FromSeconds(30), description);
+            return Retry(action, 1, description);
         }
 
-        public static void Retry(Action action, TimeSpan maxTimeout, string description)
+        public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
+        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
         {
-            Retry(action, maxTimeout, retryTime, description, f => { });
+            var start = DateTime.UtcNow;
+            var retries = 0;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (retries > maxRetries)
+                {
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+                }
+
+                try
+                {
+                    action();
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                    retries++;
+                }
+
+                Sleep(retryTime);
+            }
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
         {
-            return Retry(action, maxTimeout, retryTime, description, f => { });
-        }
+            var start = DateTime.UtcNow;
+            var retries = 0;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (retries > maxRetries)
+                {
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+                }
 
-        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
-        {
-            var r = new Retry(description, maxTimeout, retryTime, onFail);
-            r.Run(action);
-        }
+                try
+                {
+                    return action();
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                    retries++;
+                }
 
-        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
-        {
-            var r = new Retry(description, maxTimeout, retryTime, onFail);
-            return r.Run(action);
+                Sleep(retryTime);
+            }
         }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,68 +0,0 @@
-using CodexContractsPlugin.Marketplace;
-using Utils;
-
-namespace CodexContractsPlugin.ChainMonitor
-{
-    public class ChainEvents
-    {
-        private ChainEvents(
-            BlockInterval blockInterval,
-            Request[] requests,
-            RequestFulfilledEventDTO[] fulfilled,
-            RequestCancelledEventDTO[] cancelled,
-            SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
-            )
-        {
-            BlockInterval = blockInterval;
-            Requests = requests;
-            Fulfilled = fulfilled;
-            Cancelled = cancelled;
-            SlotFilled = slotFilled;
-            SlotFreed = slotFreed;
-        }
-
-        public BlockInterval BlockInterval { get; }
-        public Request[] Requests { get; }
-        public RequestFulfilledEventDTO[] Fulfilled { get; }
-        public RequestCancelledEventDTO[] Cancelled { get; }
-        public SlotFilledEventDTO[] SlotFilled { get; }
-        public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
-
-        public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
-        {
-            return FromContractEvents(contracts.GetEvents(blockInterval));
-        }
-
-        public static ChainEvents FromTimeRange(ICodexContracts contracts, TimeRange timeRange)
-        {
-            return FromContractEvents(contracts.GetEvents(timeRange));
-        }
-
-        public static ChainEvents FromContractEvents(ICodexContractsEvents events)
-        {
-            return new ChainEvents(
-                events.BlockInterval,
-                events.GetStorageRequests(),
-                events.GetRequestFulfilledEvents(),
-                events.GetRequestCancelledEvents(),
-                events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
-            );
-        }
-    }
-}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,156 +0,0 @@
-using CodexContractsPlugin.Marketplace;
-using Logging;
-using System.Numerics;
-using Utils;
-
-namespace CodexContractsPlugin.ChainMonitor
-{
-    public interface IChainStateChangeHandler
-    {
-        void OnNewRequest(IChainStateRequest request);
-        void OnRequestFinished(IChainStateRequest request);
-        void OnRequestFulfilled(IChainStateRequest request);
-        void OnRequestCancelled(IChainStateRequest request);
-        void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex);
-        void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex);
-    }
-
-    public class ChainState
-    {
-        private readonly List<ChainStateRequest> requests = new List<ChainStateRequest>();
-        private readonly ILog log;
-        private readonly ICodexContracts contracts;
-        private readonly IChainStateChangeHandler handler;
-
-        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
-        {
-            this.log = new LogPrefixer(log, "(ChainState) ");
-            this.contracts = contracts;
-            handler = changeHandler;
-            StartUtc = startUtc;
-            TotalSpan = new TimeRange(startUtc, startUtc);
-        }
-
-        public TimeRange TotalSpan { get; private set; }
-        public IChainStateRequest[] Requests => requests.ToArray();
-
-        public DateTime StartUtc { get; }
-
-        public void Update()
-        {
-            Update(DateTime.UtcNow);
-        }
-
-        public void Update(DateTime toUtc)
-        {
-            var span = new TimeRange(TotalSpan.To, toUtc);
-            var events = ChainEvents.FromTimeRange(contracts, span);
-            Apply(events);
-
-            TotalSpan = new TimeRange(TotalSpan.From, span.To);
-        }
-
-        private void Apply(ChainEvents events)
-        {
-            if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
-
-            log.Log($"ChainState updating: {events.BlockInterval}");
-
-            // Run through each block and apply the events to the state in order.
-            var span = events.BlockInterval.TimeRange.Duration;
-            var numBlocks = events.BlockInterval.NumberOfBlocks;
-            var spanPerBlock = span / numBlocks;
-
-            var eventUtc = events.BlockInterval.TimeRange.From;
-            for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++)
-            {
-                var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
-                ApplyEvents(b, blockEvents, eventUtc);
-
-                eventUtc += spanPerBlock;
-            }
-        }
-
-        private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
-        {
-            foreach (var e in blockEvents)
-            {
-                dynamic d = e;
-                ApplyEvent(d);
-            }
-
-            ApplyTimeImplicitEvents(blockNumber, eventsUtc);
-        }
-
-        private void ApplyEvent(Request request)
-        {
-            if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
-                throw new Exception("Received NewRequest event for id that already exists.");
-
-            var newRequest = new ChainStateRequest(log, request, RequestState.New);
-            requests.Add(newRequest);
-
-            handler.OnNewRequest(newRequest);
-        }
-
-        private void ApplyEvent(RequestFulfilledEventDTO request)
-        {
-            var r = FindRequest(request.RequestId);
-            if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Started);
-            handler.OnRequestFulfilled(r);
-        }
-
-        private void ApplyEvent(RequestCancelledEventDTO request)
-        {
-            var r = FindRequest(request.RequestId);
-            if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Cancelled);
-            handler.OnRequestCancelled(r);
-        }
-
-        private void ApplyEvent(SlotFilledEventDTO request)
-        {
-            var r = FindRequest(request.RequestId);
-            if (r == null) return;
-            r.Hosts.Add(request.Host, (int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFilled (host:'{request.Host}', slotIndex:{request.SlotIndex})");
-            handler.OnSlotFilled(r, request.SlotIndex);
-        }
-
-        private void ApplyEvent(SlotFreedEventDTO request)
-        {
-            var r = FindRequest(request.RequestId);
-            if (r == null) return;
-            r.Hosts.RemoveHost((int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFreed (slotIndex:{request.SlotIndex})");
-            handler.OnSlotFreed(r, request.SlotIndex);
-        }
-
-        private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
-        {
-            foreach (var r in requests)
-            {
-                if (r.State == RequestState.Started
-                    && r.FinishedUtc < eventsUtc)
-                {
-                    r.UpdateState(blockNumber, RequestState.Finished);
-                    handler.OnRequestFinished(r);
-                }
-            }
-        }
-
-        private ChainStateRequest? FindRequest(byte[] requestId)
-        {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
-            return r;
-        }
-
-        private bool Equal(byte[] a, byte[] b)
-        {
-            return a.SequenceEqual(b);
-        }
-    }
-}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -1,80 +0,0 @@
-using CodexContractsPlugin.Marketplace;
-using GethPlugin;
-using Logging;
-
-namespace CodexContractsPlugin.ChainMonitor
-{
-    public interface IChainStateRequest
-    {
-        Request Request { get; }
-        RequestState State { get; }
-        DateTime ExpiryUtc { get; }
-        DateTime FinishedUtc { get; }
-        EthAddress Client { get; }
-        RequestHosts Hosts { get; }
-    }
-
-    public class ChainStateRequest : IChainStateRequest
-    {
-        private readonly ILog log;
-
-        public ChainStateRequest(ILog log, Request request, RequestState state)
-        {
-            this.log = log;
-            Request = request;
-            State = state;
-
-            ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
-            FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
-
-            Log($"[{request.Block.BlockNumber}] Created as {State}.");
-
-            Client = new EthAddress(request.Client);
-            Hosts = new RequestHosts();
-        }
-
-        public Request Request { get; }
-        public RequestState State { get; private set; }
-        public DateTime ExpiryUtc { get; }
-        public DateTime FinishedUtc { get; }
-        public EthAddress Client { get; }
-        public RequestHosts Hosts { get; }
-
-        public void UpdateState(ulong blockNumber, RequestState newState)
-        {
-            Log($"[{blockNumber}] Transit: {State} -> {newState}");
-            State = newState;
-        }
-
-        public void Log(string msg)
-        {
-            log.Log($"Request '{Request.Id}': {msg}");
-        }
-    }
-
-    public class RequestHosts
-    {
-        private readonly Dictionary<int, EthAddress> hosts = new Dictionary<int, EthAddress>();
-
-        public void Add(EthAddress host, int index)
-        {
-            hosts.Add(index, host);
-        }
-        
-        public void RemoveHost(int index)
-        {
-            hosts.Remove(index);
-        }
-
-        public EthAddress? GetHost(int index)
-        {
-            if (!hosts.ContainsKey(index)) return null;
-            return hosts[index];
-        }
-
-        public EthAddress[] GetHosts()
-        {
-            return hosts.Values.ToArray();
-        }
-    }
-}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -1,31 +0,0 @@
-using System.Numerics;
-
-namespace CodexContractsPlugin.ChainMonitor
-{
-    public class DoNothingChainEventHandler : IChainStateChangeHandler
-    {
-        public void OnNewRequest(IChainStateRequest request)
-        {
-        }
-
-        public void OnRequestCancelled(IChainStateRequest request)
-        {
-        }
-
-        public void OnRequestFinished(IChainStateRequest request)
-        {
-        }
-
-        public void OnRequestFulfilled(IChainStateRequest request)
-        {
-        }
-
-        public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex)
-        {
-        }
-
-        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
-        {
-        }
-    }
-}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,10 +2,9 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -20,13 +19,15 @@ namespace CodexContractsPlugin
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
 
-        ICodexContractsEvents GetEvents(TimeRange timeRange);
-        ICodexContractsEvents GetEvents(BlockInterval blockInterval);
+        Request[] GetStorageRequests(BlockInterval blockRange);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
     }
 
-    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -62,7 +63,7 @@ namespace CodexContractsPlugin
 
         public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return StartInteraction().MintTestTokens(ethAddress, testTokens.TstWei, Deployment.TokenAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -73,17 +74,68 @@ namespace CodexContractsPlugin
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
-            return balance.TstWei();
+            return balance.TestTokens();
         }
 
-        public ICodexContractsEvents GetEvents(TimeRange timeRange)
+        public Request[] GetStorageRequests(BlockInterval blockRange)
         {
-            return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange));
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, blockRange);
+            var i = StartInteraction();
+            return events
+                    .Select(e =>
+                    {
+                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
         }
 
-        public ICodexContractsEvents GetEvents(BlockInterval blockInterval)
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange)
         {
-            return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange)
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, blockRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange)
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange)
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, blockRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
         }
 
         public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
@@ -114,6 +166,12 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,108 +0,0 @@
-using CodexContractsPlugin.Marketplace;
-using GethPlugin;
-using Logging;
-using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
-using Utils;
-
-namespace CodexContractsPlugin
-{
-    public interface ICodexContractsEvents
-    {
-        BlockInterval BlockInterval { get; }
-        Request[] GetStorageRequests();
-        RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
-        RequestCancelledEventDTO[] GetRequestCancelledEvents();
-        SlotFilledEventDTO[] GetSlotFilledEvents();
-        SlotFreedEventDTO[] GetSlotFreedEvents();
-    }
-
-    public class CodexContractsEvents : ICodexContractsEvents
-    {
-        private readonly ILog log;
-        private readonly IGethNode gethNode;
-        private readonly CodexContractsDeployment deployment;
-
-        public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval)
-        {
-            this.log = log;
-            this.gethNode = gethNode;
-            this.deployment = deployment;
-            BlockInterval = blockInterval;
-        }
-        
-        public BlockInterval BlockInterval { get; }
-
-        public Request[] GetStorageRequests()
-        {
-            var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
-            {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
-        }
-
-        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
-        {
-            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
-        }
-
-        public RequestCancelledEventDTO[] GetRequestCancelledEvents()
-        {
-            var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFilledEventDTO[] GetSlotFilledEvents()
-        {
-            var events = gethNode.GetEvents<SlotFilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFreedEventDTO[] GetSlotFreedEvents()
-        {
-            var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
-        }
-
-        private BlockTimeEntry GetBlock(ulong number)
-        {
-            return gethNode.GetBlockForNumber(number);
-        }
-
-        private EthAddress GetEthAddressFromTransaction(string transactionHash)
-        {
-            var transaction = gethNode.GetTransaction(transactionHash);
-            return new EthAddress(transaction.From);
-        }
-    }
-}

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,7 +59,7 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            }, nameof(DeployContract));
+            });
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@@ -71,7 +71,7 @@ namespace CodexContractsPlugin
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
 
             Log("Synced. Codex SmartContracts deployed.");
 
@@ -83,9 +83,9 @@ namespace CodexContractsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate, string msg)
+        private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -44,7 +44,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        public string MintTestTokens(EthAddress address, BigInteger amount, string tokenAddress)
+        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             return MintTokens(address.Address, amount, tokenAddress);
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private string MintTokens(string account, BigInteger amount, string tokenAddress)
+        private string MintTokens(string account, decimal amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -93,7 +93,7 @@ namespace CodexContractsPlugin
             var function = new MintTokensFunction
             {
                 Holder = account,
-                Amount = amount
+                Amount = amount.ToBig()
             };
 
             return gethNode.SendTransaction(tokenAddress, function);

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,5 +1,4 @@
-using CodexContractsPlugin.Marketplace;
-using KubernetesWorkflow;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -54,18 +53,7 @@ namespace CodexContractsPlugin
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            var byteCode = artifact["bytecode"];
-            var abiResult = abi!.ToString(Formatting.None);
-            var byteCodeResult = byteCode!.ToString(Formatting.None);
-
-            if (byteCodeResult
-                .ToLowerInvariant()
-                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
-            {
-                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
-            }
-
-            return abiResult;
+            return abi!.ToString(Formatting.None);
         }
 
         private static string Retry(Func<string> fetch)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,56 +1,41 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
 {
-    public interface IHasBlock
-    {
-        BlockTimeEntry Block { get; set; }
-    }
-
-    public partial class Request : RequestBase, IHasBlock
+    public partial class Request : RequestBase
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
-
-        [JsonIgnore]
-        public string Id
-        {
-            get
-            {
-                return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant();
-            }
-        }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -1,14 +1 @@
 This code was generated using the Nethereum code generator, here: http://playground.nethereum.com
-
-1. Go to site -> Abi Code Gen.
-1. Contract name = "Marketplace".
-1. In container, get "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json".
-1. Save only ABI section as new JSON. (top-level is a json array.)
-1. From original JSON get byte code.
-1. Put ABI JSON and byte code into site.
-1. Generate.
-1. From site generated code, copy `public partial class MarketplaceDeployment` and everything after it. (be considerate of namespace brackets!)
-1. In Marketplace/Marketplace.cs, replace content of 'namespace CodexContractsPlugin.Marketplace'.
-
-
-

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -1,102 +1,45 @@
-using System.Numerics;
-
-namespace CodexContractsPlugin
+namespace CodexContractsPlugin
 {
     public class TestToken : IComparable<TestToken>
     {
-        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
-
-        public TestToken(BigInteger tstWei)
+        public TestToken(decimal amount)
         {
-            TstWei = tstWei;
-            Tst = tstWei / WeiFactor;
+            Amount = amount;
         }
 
-        public BigInteger TstWei { get; }
-        public BigInteger Tst { get; }
+        public decimal Amount { get; }
 
         public int CompareTo(TestToken? other)
         {
-            return TstWei.CompareTo(other!.TstWei);
+            return Amount.CompareTo(other!.Amount);
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is TestToken token && TstWei == token.TstWei;
+            return obj is TestToken token && Amount == token.Amount;
         }
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(TstWei);
+            return HashCode.Combine(Amount);
         }
 
         public override string ToString()
         {
-            var weiOnly = TstWei % WeiFactor;
-
-            var tokens = new List<string>();
-            if (Tst > 0) tokens.Add($"{Tst} TST");
-            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
-
-            return string.Join(" + ", tokens);
-        }
-
-        public static TestToken operator +(TestToken a, TestToken b)
-        {
-            return new TestToken(a.TstWei + b.TstWei);
-        }
-
-        public static bool operator <(TestToken a, TestToken b)
-        {
-            return a.TstWei < b.TstWei;
-        }
-
-        public static bool operator >(TestToken a, TestToken b)
-        {
-            return a.TstWei > b.TstWei;
-        }
-
-        public static bool operator ==(TestToken a, TestToken b)
-        {
-            return a.TstWei == b.TstWei;
-        }
-
-        public static bool operator !=(TestToken a, TestToken b)
-        {
-            return a.TstWei != b.TstWei;
+            return $"{Amount} TestTokens";
         }
     }
 
-    public static class TestTokensExtensions
+    public static class TokensIntExtensions
     {
-        public static TestToken TstWei(this int i)
+        public static TestToken TestTokens(this int i)
         {
-            return TstWei(Convert.ToDecimal(i));
+            return TestTokens(Convert.ToDecimal(i));
         }
 
-        public static TestToken TstWei(this decimal i)
-        {
-            return new TestToken(new BigInteger(i));
-        }
-
-        public static TestToken TstWei(this BigInteger i)
+        public static TestToken TestTokens(this decimal i)
         {
             return new TestToken(i);
         }
-
-        public static TestToken Tst(this int i)
-        {
-            return Tst(Convert.ToDecimal(i));
-        }
-
-        public static TestToken Tst(this decimal i)
-        {
-            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
-        }
-
-        public static TestToken Tst(this BigInteger i)
-        {
-            return new TestToken(i * TestToken.WeiFactor);
-        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,13 +1,11 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
-using Utils;
 
 namespace CodexDiscordBotPlugin
 {
     public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
-        private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!";
         private readonly IPluginTools tools;
 
         public CodexDiscordBotPlugin(IPluginTools tools)
@@ -31,76 +29,31 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningPod Deploy(DiscordBotStartupConfig config)
+        public RunningContainers Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
+        public RunningContainers DeployRewarder(RewarderBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartRewarderContainer(workflow, config);
         }
 
-        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
-            WaitForStartupMessage(workflow, pod);
-            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
-            return pod;
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
         }
 
-        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
-            startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            var pod = workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
-            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
-            return pod;
-        }
-
-        private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod)
-        {
-            var finder = new LogLineFinder(ExpectedStartupMessage, workflow);
-            Time.WaitUntil(() =>
-            {
-                finder.FindLine(pod);
-                return finder.Found;
-            }, nameof(WaitForStartupMessage));
-        }
-
-        public class LogLineFinder : LogHandler
-        {
-            private readonly string message;
-            private readonly IStartupWorkflow workflow;
-
-            public LogLineFinder(string message, IStartupWorkflow workflow)
-            {
-                this.message = message;
-                this.workflow = workflow;
-            }
-
-            public void FindLine(RunningPod pod)
-            {
-                Found = false;
-                foreach (var c in pod.Containers)
-                {
-                    workflow.DownloadContainerLog(c, this);
-                    if (Found) return;
-                }
-            }
-
-            public bool Found { get; private set; }
-
-            protected override void ProcessLine(string line)
-            {
-                if (!Found && line.Contains(message)) Found = true;
-            }
+            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig);
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
-        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
         {
             return Plugin(ci).DeployRewarder(config);
         }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "codexstorage/codex-discordbot:sha-22cf82b";
+        public override string Image => "codexstorage/codex-discordbot:sha-8c64352";
 
         public static string RewardsPort = "bot_rewards_port";
 
@@ -33,8 +33,6 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
             AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
-            AddEnvVar("NODISCORD", "1");
-
             AddInternalPortAndVar("REWARDAPIPORT", RewardsPort);
 
             if (!string.IsNullOrEmpty(config.DataPath))

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -27,9 +27,8 @@
 
     public class RewarderBotStartupConfig
     {
-        public RewarderBotStartupConfig(string name, string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
+        public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
         {
-            Name = name;
             DiscordBotHost = discordBotHost;
             DiscordBotPort = discordBotPort;
             IntervalMinutes = intervalMinutes;
@@ -38,10 +37,9 @@
             DataPath = dataPath;
         }
 
-        public string Name { get; }
         public string DiscordBotHost { get; }
         public int DiscordBotPort { get; }
-        public int IntervalMinutes { get; }
+        public string IntervalMinutes { get; }
         public DateTime HistoryStartUtc { get; }
         public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "thatbenbierens/codex-rewardbot:newstate";
-            //"codexstorage/codex-rewarderbot:sha-12dc7ef";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -18,7 +17,7 @@ namespace CodexDiscordBotPlugin
 
             AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost);
             AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString());
-            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes.ToString());
+            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes);
             var offset = new DateTimeOffset(config.HistoryStartUtc);
             AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString());
 

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "5A-B0-2A-AC-42-B1-A2-49-6F-9D-4E-D8-56-40-10-A6-67-F4-0D-2A-9F-E0-84-5C-EB-B8-2D-4F-D8-56-79-6C";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -38,7 +38,7 @@ namespace CodexPlugin
             if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
         }
 
-        public void CheckCompatibility(RunningPod[] containers)
+        public void CheckCompatibility(RunningContainers[] containers)
         {
             if (checkPassed) return;
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -2,29 +2,28 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
-using Logging;
 using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
 {
-    public class CodexAccess
+    public class CodexAccess : ILogHandler
     {
-        private readonly ILog log;
         private readonly IPluginTools tools;
         private readonly Mapper mapper = new Mapper();
+        private bool hasContainerCrashed;
 
-        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
+        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
-            log = tools.GetLog();
             Container = container;
             CrashWatcher = crashWatcher;
+            hasContainerCrashed = false;
 
-            CrashWatcher.Start();
+            CrashWatcher.Start(this);
         }
 
-        public RunningPod Container { get; }
+        public RunningContainer Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
         public DebugInfo GetDebugInfo()
@@ -35,23 +34,20 @@ namespace CodexPlugin
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
-            return CrashCheck(() =>
+            var endpoint = GetEndpoint();
+            var str = endpoint.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
             {
-                var endpoint = GetEndpoint();
-                var str = endpoint.HttpGetString($"debug/peer/{peerId}");
-
-                if (str.ToLowerInvariant() == "unable to find peer!")
+                return new DebugPeer
                 {
-                    return new DebugPeer
-                    {
-                        IsPeerFound = false
-                    };
-                }
+                    IsPeerFound = false
+                };
+            }
 
-                var result = endpoint.Deserialize<DebugPeer>(str);
-                result.IsPeerFound = true;
-                return result;
-            });
+            var result = endpoint.Deserialize<DebugPeer>(str);
+            result.IsPeerFound = true;
+            return result;
         }
 
         public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
@@ -63,19 +59,14 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(FileStream fileStream)
         {
-            return OnCodex(
-                api => api.UploadAsync(fileStream),
-                CreateRetryConfig(nameof(UploadFile), onFailure));
+            return OnCodex(api => api.UploadAsync(fileStream));
         }
 
-        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
+        public Stream DownloadFile(string contentId)
         {
-            var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
-                CreateRetryConfig(nameof(DownloadFile), onFailure));
-
+            var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId));
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
@@ -98,24 +89,15 @@ namespace CodexPlugin
             return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
-        public CodexSpace Space()
-        {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
-            return mapper.Map(space);
-        }
-
         public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            return CrashCheck(() =>
+            var endpoint = GetEndpoint();
+            return Time.Retry(() =>
             {
-                var endpoint = GetEndpoint();
-                return Time.Retry(() =>
-                {
-                    var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
-                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                    return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                }, nameof(GetPurchaseStatus));
-            });
+                var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+            }, nameof(GetPurchaseStatus));
 
             // TODO: current getpurchase api does not line up with its openapi spec.
             // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
@@ -132,65 +114,17 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
-        public void LogDiskSpace(string msg)
-        {
-            try
-            {
-                var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
-                Log($"{msg} - Disk info: {diskInfo}");
-            }
-            catch (Exception e)
-            {
-                Log("Failed to get disk info: " + e);
-            }
-        }
-
-        public void DeleteRepoFolder()
-        {
-            try
-            {
-                var containerNumber = Container.Containers.First().Recipe.Number;
-                var dataDir = $"datadir{containerNumber}";
-                var workflow = tools.CreateWorkflow();
-                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
-                Log("Deleted repo folder.");
-            }
-            catch (Exception e)
-            {
-                Log("Unable to delete repo folder: " + e);
-            }
-        }
-
         private T OnCodex<T>(Func<CodexApi, Task<T>> action)
-        {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action));
-            return result;
-        }
-
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
-        {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
-            return result;
-        }
-
-        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
         {
             var address = GetAddress();
-            var api = new CodexApi(client);
-            api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
-            return CrashCheck(() => Time.Wait(action(api)));
-        }
-
-        private T CrashCheck<T>(Func<T> action)
-        {
-            try
+            var result = tools.CreateHttp(CheckContainerCrashed)
+                .OnClient(client =>
             {
-                return action();
-            }
-            finally
-            {
-                CrashWatcher.HasContainerCrashed();
-            }
+                var api = new CodexApi(client);
+                api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+                return Time.Wait(action(api));
+            });
+            return result;
         }
 
         private IEndpoint GetEndpoint()
@@ -202,67 +136,31 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
         {
-            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
+            if (hasContainerCrashed) throw new Exception("Container has crashed.");
         }
 
-        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
+        public void Log(Stream crashLog)
         {
-            var timeSet = tools.TimeSet;
+            var log = tools.GetLog();
+            var file = log.CreateSubfile();
+            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+            file.Write($"Container Crash Log for {Container.Name}.");
 
-            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
+            using var reader = new StreamReader(crashLog);
+            var line = reader.ReadLine();
+            while (line != null)
             {
-                onFailure(failure);
-                Investigate(failure, timeSet);
-            });
-        }
-
-        private void Investigate(Failure failure, ITimeSet timeSet)
-        {
-            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
-                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
-                $"Checking if node responds to debug/info...");
-
-            LogDiskSpace("After retry failure");
-
-            try
-            {
-                var debugInfo = GetDebugInfo();
-                if (string.IsNullOrEmpty(debugInfo.Spr))
-                {
-                    Log("Did not get value debug/info response.");
-                    Throw(failure);
-                }
-                else
-                {
-                    Log("Got valid response from debug/info.");
-                }
-            }
-            catch (Exception ex)
-            {
-                Log("Got exception from debug/info call: " + ex);
-                Throw(failure);
+                file.Write(line);
+                line = reader.ReadLine();
             }
 
-            if (failure.Duration < timeSet.HttpCallTimeout())
-            {
-                Log("Retry failed within HTTP timeout duration.");
-                Throw(failure);
-            }
-        }
-
-        private void Throw(Failure failure)
-        {
-            throw failure.Exception;
-        }
-
-        private void Log(string msg)
-        {
-            log.Log($"{GetName()} {msg}");
+            log.Log("Crash log successfully downloaded.");
+            hasContainerCrashed = true;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1524803-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -109,9 +108,8 @@ namespace CodexPlugin
 
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
-                Additional(account);
+                AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
+                Additional(marketplaceSetup.EthAccount);
 
                 SetCommandOverride(marketplaceSetup);
                 if (marketplaceSetup.IsValidator)
@@ -120,7 +118,7 @@ namespace CodexPlugin
                 }
             }
 
-            if (!string.IsNullOrEmpty(config.NameOverride))
+            if(!string.IsNullOrEmpty(config.NameOverride))
             {
                 AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
@@ -160,7 +158,7 @@ namespace CodexPlugin
 
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)
         {
-            if (config.StorageQuota != null) return config.StorageQuota.Multiply(1.2);
+            if (config.StorageQuota != null) return config.StorageQuota;
             // Default Codex quota: 8 Gb, using +20% to be safe.
             return 8.GB().Multiply(1.2);
         }

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
     public class CodexDeployment
     {
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
-            CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
-            RunningPod? discordBotContainer, DeploymentMetadata metadata,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
             String id)
         {
             Id = id;
@@ -24,20 +24,20 @@ namespace CodexPlugin
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningPod? PrometheusContainer { get; }
-        public RunningPod? DiscordBotContainer { get; }
+        public RunningContainers? PrometheusContainer { get; }
+        public RunningContainers? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningPod pod, DebugInfo info)
+        public CodexInstance(RunningContainers containers, DebugInfo info)
         {
-            Pod = pod;
+            Containers = containers;
             Info = info;
         }
 
-        public RunningPod Pod { get; }
+        public RunningContainers Containers { get; }
         public DebugInfo Info { get; }
     }
 

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -1,105 +0,0 @@
-using System.Globalization;
-
-namespace CodexPlugin
-{
-    public class CodexLogLine
-    {
-        public static CodexLogLine? Parse(string line)
-        {
-            try
-            {
-                if (string.IsNullOrEmpty(line) ||
-                    line.Length < 34 ||
-                    line[3] != ' ' ||
-                    line[33] != ' ') return null;
-
-                line = line.Replace(Environment.NewLine, string.Empty);
-
-                var level = line.Substring(0, 3);
-                var dtLine = line.Substring(4, 23);
-
-                var firstEqualSign = line.IndexOf('=');
-                var msgStart = 34;
-                var msgEnd = line.Substring(0, firstEqualSign).LastIndexOf(' ');
-                var msg = line.Substring(msgStart, msgEnd - msgStart).Trim();
-                var attrsLine = line.Substring(msgEnd);
-
-                var attrs = SplitAttrs(attrsLine);
-
-                var format = "yyyy-MM-dd HH:mm:ss.fff";
-                var dt = DateTime.ParseExact(dtLine, format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime();
-
-                return new CodexLogLine()
-                {
-                    LogLevel = level,
-                    TimestampUtc = dt,
-                    Message = msg,
-                    Attributes = attrs
-                };
-            }
-            catch
-            {
-                return null;
-            }
-        }
-
-        public string LogLevel { get; set; } = string.Empty;
-        public DateTime TimestampUtc { get; set; }
-        public string Message { get; set; } = string.Empty;
-        public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
-
-        /// <summary>
-        /// After too much time spent cursing at regexes, here's what I got:
-        /// Parses input string into 'key=value' pair, considerate of quoted (") values.
-        /// </summary>
-        private static Dictionary<string, string> SplitAttrs(string input)
-        {
-            input += " ";
-            var result = new Dictionary<string, string>();
-
-            var key = string.Empty;
-            var value = string.Empty;
-            var mode = 1;
-            var inQuote = false;
-
-            foreach (var c in input)
-            {
-                if (mode == 1)
-                {
-                    if (c == '=') mode = 2;
-                    else if (c == ' ')
-                    {
-                        if (string.IsNullOrEmpty(key)) continue;
-                        else
-                        {
-                            result.Add(key, string.Empty);
-                            key = string.Empty;
-                            value = string.Empty;
-                        }
-                    }
-                    else key += c;
-                }
-                else if (mode == 2)
-                {
-                    if (c == ' ' && !inQuote)
-                    {
-                        result.Add(key, value);
-                        key = string.Empty;
-                        value = string.Empty;
-                        mode = 1;
-                    }
-                    else if (c == '\"')
-                    {
-                        inQuote = !inQuote;
-                    }
-                    else
-                    {
-                        value += c;
-                    }
-                }
-            }
-
-            return result;
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -15,24 +15,14 @@ namespace CodexPlugin
         DebugInfo GetDebugInfo();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
-        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
-        CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
         DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
-        EthAccount EthAccount { get; }
-
-        /// <summary>
-        /// Warning! The node is not usable after this.
-        /// TODO: Replace with delete-blocks debug call once available in Codex.
-        /// </summary>
-        void DeleteRepoFolder();
         void Stop(bool waitTillStopped);
     }
 
@@ -40,13 +30,13 @@ namespace CodexPlugin
     {
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
-        private readonly EthAccount? ethAccount;
+        private readonly EthAddress? ethAddress;
         private readonly TransferSpeeds transferSpeeds;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
         {
             this.tools = tools;
-            this.ethAccount = ethAccount;
+            this.ethAddress = ethAddress;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
@@ -54,9 +44,7 @@ namespace CodexPlugin
             transferSpeeds = new TransferSpeeds();
         }
 
-        public RunningPod Pod { get { return CodexAccess.Container; } }
-        
-        public RunningContainer Container { get { return Pod.Containers.Single(); } }
+        public RunningContainer Container { get { return CodexAccess.Container; } }
         public CodexAccess CodexAccess { get; }
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
@@ -68,7 +56,7 @@ namespace CodexPlugin
         {
             get
             {
-                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
             }
         }
 
@@ -76,30 +64,21 @@ namespace CodexPlugin
         {
             get
             {
-                EnsureMarketplace();
-                return ethAccount!.EthAddress;
-            }
-        }
-
-        public EthAccount EthAccount
-        {
-            get
-            {
-                EnsureMarketplace();
-                return ethAccount!;
+                if (ethAddress == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
+                return ethAddress;
             }
         }
 
         public string GetName()
         {
-            return Container.Name;
+            return CodexAccess.Container.Name;
         }
 
         public DebugInfo GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
             var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}");
             return debugInfo;
         }
 
@@ -110,20 +89,13 @@ namespace CodexPlugin
 
         public ContentId UploadFile(TrackedFile file)
         {
-            return UploadFile(file, DoNothing);
-        }
-
-        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
-        {
-            CodexAccess.LogDiskSpace("Before upload");
-
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
             var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(fileStream);
             });
 
             var response = measurement.Value;
@@ -133,22 +105,15 @@ namespace CodexPlugin
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
             Log($"Uploaded file. Received contentId: '{response}'.");
-            CodexAccess.LogDiskSpace("After upload");
-
             return new ContentId(response);
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
-        {
-            return DownloadContent(contentId, DoNothing, fileLabel);
-        }
-
-        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
             transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
@@ -159,11 +124,6 @@ namespace CodexPlugin
             return CodexAccess.LocalFiles();
         }
 
-        public CodexSpace Space()
-        {
-            return CodexAccess.Space();
-        }
-
         public void ConnectToPeer(ICodexNode node)
         {
             var peer = (CodexNode)node;
@@ -180,16 +140,13 @@ namespace CodexPlugin
             return CodexAccess.GetPodInfo();
         }
 
-        public void DeleteRepoFolder()
-        {
-            CodexAccess.DeleteRepoFolder();
-        }
-
         public void Stop(bool waitTillStopped)
         {
-            Log("Stopping...");
-            CrashWatcher.Stop();
-            Group.Stop(this, waitTillStopped);
+            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+                "available for codex-nodes in groups of 1.");
+
+            Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
@@ -214,21 +171,19 @@ namespace CodexPlugin
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Pod);
+            var podInfo = workflow.GetPodInfo(peer.Container);
 
             return peerInfo.Addrs.Select(a => a
                 .Replace("0.0.0.0", podInfo.Ip))
                 .ToArray();
         }
 
-        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
+        private void DownloadToFile(string contentId, TrackedFile file)
         {
-            CodexAccess.LogDiskSpace("Before download");
-
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                using var downloadStream = CodexAccess.DownloadFile(contentId);
                 downloadStream.CopyTo(fileStream);
             }
             catch
@@ -236,22 +191,11 @@ namespace CodexPlugin
                 Log($"Failed to download file '{contentId}'.");
                 throw;
             }
-
-            CodexAccess.LogDiskSpace("After download");
-        }
-
-        private void EnsureMarketplace()
-        {
-            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
         }
 
         private void Log(string msg)
         {
             tools.GetLog().Log($"{GetName()}: {msg}");
         }
-
-        private void DoNothing(Failure failure)
-        {
-        }
     }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -22,22 +22,22 @@ namespace CodexPlugin
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
-            var ethAccount = GetEthAccount(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
+            var ethAddress = GetEthAddress(access);
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAddress);
+            return new CodexNode(tools, access, group, marketplaceAccess, ethAddress);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAddress? ethAddress)
         {
-            if (ethAccount == null) return new MarketplaceUnavailable();
+            if (ethAddress == null) return new MarketplaceUnavailable();
             return new MarketplaceAccess(tools.GetLog(), codexAccess);
         }
 
-        private EthAccount? GetEthAccount(CodexAccess access)
+        private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
+            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
-            return ethAccount;
+            return ethAccount.EthAddress;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -15,11 +15,11 @@ namespace CodexPlugin
     {
         private readonly CodexStarter starter;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.starter = starter;
             Containers = containers;
-            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
             Version = new DebugInfoVersion();
         }
 
@@ -39,14 +39,7 @@ namespace CodexPlugin
             Containers = null!;
         }
 
-        public void Stop(CodexNode node, bool waitTillStopped)
-        {
-            starter.Stop(node.Pod, waitTillStopped);
-            Nodes = Nodes.Where(n => n != node).ToArray();
-            Containers = Containers.Where(c => c != node.Pod).ToArray();
-        }
-
-        public RunningPod[] Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
         public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
@@ -81,9 +74,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
+        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
+            var watcher = factory.CreateCrashWatcher(c);
             var access = new CodexAccess(tools, c, watcher);
             return factory.CreateOnlineCodexNode(access, this);
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -32,13 +32,13 @@ namespace CodexPlugin
         {
         }
 
-        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
         {
             containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
             return codexStarter.WrapCodexContainers(coreInterface, containers);

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -52,7 +52,6 @@ namespace CodexPlugin
         public CodexLogLevel Libp2p { get; set; }
         public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
-        public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn;
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -168,8 +167,8 @@ namespace CodexPlugin
         public bool IsStorageNode { get; private set; }
         public bool IsValidator { get; private set; }
         public Ether InitialEth { get; private set; } = 0.Eth();
-        public TestToken InitialTestTokens { get; private set; } = 0.Tst();
-        public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup();
+        public TestToken InitialTestTokens { get; private set; } = 0.TestTokens();
+        public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew();
 
         public IMarketplaceSetup AsStorageNode()
         {
@@ -185,7 +184,7 @@ namespace CodexPlugin
 
         public IMarketplaceSetup WithAccount(EthAccount account)
         {
-            EthAccountSetup.Pin(account);
+            EthAccount = account;
             return this;
         }
 
@@ -201,41 +200,10 @@ namespace CodexPlugin
             var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
             result += IsStorageNode ? "(storageNode)" : "()";
             result += IsValidator ? "(validator)" : "() ";
-            result += $"Address: '{EthAccountSetup}' ";
-            result += $"{InitialEth.Eth} / {InitialTestTokens}";
+            result += $"Address: '{EthAccount.EthAddress}' ";
+            result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})";
             result += "] ";
             return result;
         }
     }
-
-    public class EthAccountSetup
-    {
-        private readonly List<EthAccount> accounts = new List<EthAccount>();
-        private bool pinned = false;
-
-        public void Pin(EthAccount account)
-        {
-            accounts.Add(account);
-            pinned = true;
-        }
-
-        public EthAccount GetNew()
-        {
-            if (pinned) return accounts.Last();
-
-            var a = EthAccount.GenerateNew();
-            accounts.Add(a);
-            return a;
-        }
-
-        public EthAccount[] GetAll()
-        {
-            return accounts.ToArray();
-        }
-
-        public override string ToString()
-        {
-            return string.Join(",", accounts.Select(a => a.ToString()).ToArray());
-        }
-    }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -19,7 +19,7 @@ namespace CodexPlugin
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public RunningPod[] BringOnline(CodexSetup codexSetup)
+        public RunningContainers[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -33,15 +33,15 @@ namespace CodexPlugin
             foreach (var rc in containers)
             {
                 var podInfo = GetPodInfo(rc);
-                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' PodLabel: '{c.RunningPod.StartResult.Deployment.PodLabel}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
+                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();
 
             return containers;
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
         {
             var codexNodeFactory = new CodexNodeFactory(pluginTools);
 
@@ -65,14 +65,6 @@ namespace CodexPlugin
             Log("Stopped.");
         }
 
-        public void Stop(RunningPod pod, bool waitTillStopped)
-        {
-            Log($"Stopping node...");
-            var workflow = pluginTools.CreateWorkflow();
-            workflow.Stop(pod, waitTillStopped);
-            Log("Stopped.");
-        }
-
         public string GetCodexId()
         {
             if (versionResponse != null) return versionResponse.Version;
@@ -93,27 +85,24 @@ namespace CodexPlugin
             return startupConfig;
         }
 
-        private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var futureContainers = new List<FutureContainers>();
+            var result = new List<RunningContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
+                result.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-
-            return futureContainers
-                .Select(f => f.WaitForOnline())
-                .ToArray();
+            return result.ToArray();
         }
 
-        private PodInfo GetPodInfo(RunningPod rc)
+        private PodInfo GetPodInfo(RunningContainers rc)
         {
             var workflow = pluginTools.CreateWorkflow();
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
 
@@ -130,10 +119,10 @@ namespace CodexPlugin
             return group;
         }
 
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
+        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
         {
             Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
+            foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
         }
 
         private void LogSeparator()

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -73,18 +73,11 @@ namespace CodexPlugin
                     "contracts",
                     "clock"
                 };
-                var jsonSerializeTopics = new[]
-                {
-                    "serde",
-                    "json",
-                    "serialization"
-                };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
-                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
-                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)}";
+                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -105,18 +105,4 @@ namespace CodexPlugin
             return HashCode.Combine(Id);
         }
     }
-
-    public class CodexSpace
-    {
-        public long TotalBlocks { get; set; }
-        public long QuotaMaxBytes { get; set; }
-        public long QuotaUsedBytes { get; set; }
-        public long QuotaReservedBytes { get; set; }
-        public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-    }
 }

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);
         }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -1,5 +1,4 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Newtonsoft.Json.Linq;
 using System.Numerics;
 using Utils;
@@ -57,78 +56,34 @@ namespace CodexPlugin
                 ProofProbability = ToDecInt(purchase.ProofProbability),
                 Reward = ToDecInt(purchase.PricePerSlotPerSecond),
                 Collateral = ToDecInt(purchase.RequiredCollateral),
-                Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
+                Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds),
                 Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
                 Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };
         }
 
-        // TODO: Fix openapi spec for this call.
-        //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
-        //{
-        //    return new StoragePurchase(Map(purchase.Request))
-        //    {
-        //        State = purchase.State,
-        //        Error = purchase.Error
-        //    };
-        //}
-
-        //public StorageRequest Map(CodexOpenApi.StorageRequest request)
-        //{
-        //    return new StorageRequest(Map(request.Ask), Map(request.Content))
-        //    {
-        //        Id = request.Id,
-        //        Client = request.Client,
-        //        Expiry = TimeSpan.FromSeconds(Convert.ToInt64(request.Expiry)),
-        //        Nonce = request.Nonce
-        //    };
-        //}
-
-        //public StorageAsk Map(CodexOpenApi.StorageAsk ask)
-        //{
-        //    return new StorageAsk
-        //    {
-        //        Duration = TimeSpan.FromSeconds(Convert.ToInt64(ask.Duration)),
-        //        MaxSlotLoss = ask.MaxSlotLoss,
-        //        ProofProbability = ask.ProofProbability,
-        //        Reward = Convert.ToDecimal(ask.Reward).TstWei(),
-        //        Slots = ask.Slots,
-        //        SlotSize = new ByteSize(Convert.ToInt64(ask.SlotSize))
-        //    };
-        //}
-
-        //public StorageContent Map(CodexOpenApi.Content content)
-        //{
-        //    return new StorageContent
-        //    {
-        //        Cid = content.Cid
-        //    };
-        //}
+        public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        {
+            return new StoragePurchase
+            {
+                State = purchase.State,
+                Error = purchase.Error
+            };
+        }
 
         public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
         {
             return new StorageAvailability(
-                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
+                totalSpace: new Utils.ByteSize(Convert.ToInt64(read.TotalSize)),
                 maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
-                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
+                minPriceForTotalSpace: new TestToken(Convert.ToDecimal(read.MinPrice)),
+                maxCollateral: new TestToken(Convert.ToDecimal(read.MaxCollateral))
             )
             {
                 Id = read.Id
             };
         }
 
-        public CodexSpace Map(Space space)
-        {
-            return new CodexSpace
-            {
-                QuotaMaxBytes = space.QuotaMaxBytes,
-                QuotaReservedBytes = space.QuotaReservedBytes,
-                QuotaUsedBytes = space.QuotaUsedBytes,
-                TotalBlocks = space.TotalBlocks
-            };
-        }
-
         private DebugInfoVersion MapDebugInfoVersion(JObject obj)
         {
             return new DebugInfoVersion
@@ -143,7 +98,7 @@ namespace CodexPlugin
             return new DebugInfoTable
             {
                 LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                Nodes = new DebugInfoTableNode[0]
             };
         }
 
@@ -162,16 +117,6 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
-        {
-            if (nodes == null || nodes.Count == 0)
-            {
-                return new DebugInfoTableNode[0];
-            }
-
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
-        }
-
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
         {
             return new Manifest
@@ -220,7 +165,8 @@ namespace CodexPlugin
 
         private string ToDecInt(TestToken t)
         {
-            return t.TstWei.ToString("D");
+            var i = new BigInteger(t.Amount);
+            return i.ToString("D");
         }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
@@ -6,7 +7,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
-        IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
+        StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -20,7 +21,7 @@ namespace CodexPlugin
             this.codexAccess = codexAccess;
         }
 
-        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
 
@@ -37,9 +38,7 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase);
-            contract.WaitForStorageContractSubmitted();
-            return contract;
+            return new StoragePurchaseContract(log, codexAccess, response, purchase);
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
@@ -55,7 +54,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
+            log.Log($"{codexAccess.Container.Name} {msg}");
         }
     }
 
@@ -67,7 +66,7 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
-        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             Unavailable();
             throw new NotImplementedException();
@@ -79,4 +78,78 @@ namespace CodexPlugin
             throw new InvalidOperationException();
         }
     }
+
+    public class StoragePurchaseContract
+    {
+        private readonly ILog log;
+        private readonly CodexAccess codexAccess;
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10);
+        private DateTime? contractStartUtc;
+
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            Purchase = purchase;
+        }
+
+        public string PurchaseId { get; }
+        public StoragePurchaseRequest Purchase { get; }
+
+        public void WaitForStorageContractStarted()
+        {
+            var timeout = Purchase.Expiry + gracePeriod;
+
+            WaitForStorageContractState(timeout, "started");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+        }
+
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+
+            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.State != lastState)
+                {
+                    lastState = purchaseStatus.State;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(1000);
+
+                if (lastState == "errored")
+                {
+                    FrameworkAssert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
+                }
+            }
+            log.Log($"Contract '{desiredState}'.");
+        }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,7 +1,5 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Logging;
-using System.Data;
 using Utils;
 
 namespace CodexPlugin
@@ -14,8 +12,8 @@ namespace CodexPlugin
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
-        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
+        public TestToken PricePerSlotPerSecond { get; set; } = 1.TestTokens();
+        public TestToken RequiredCollateral { get; set; } = 1.TestTokens();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }
@@ -39,34 +37,6 @@ namespace CodexPlugin
     {
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
-        public StorageRequest Request { get; set; } = null!;
-    }
-
-    public class StorageRequest
-    {
-        public string Id { get; set; } = string.Empty;
-        public string Client { get; set; } = string.Empty;
-        public StorageAsk Ask { get; set; } = null!;
-        public StorageContent Content { get; set; } = null!;
-        public string Expiry { get; set; } = string.Empty;
-        public string Nonce { get; set; } = string.Empty;
-    }
-
-    public class StorageAsk
-    {
-        public int Slots { get; set; }
-        public string SlotSize { get; set; } = string.Empty;
-        public string Duration { get; set; } = string.Empty;
-        public string ProofProbability { get; set; } = string.Empty;
-        public string Reward { get; set; } = string.Empty;
-        public int MaxSlotLoss { get; set; }
-    }
-
-    public class StorageContent
-    {
-        public string Cid { get; set; } = string.Empty;
-        //public ErasureParameters Erasure { get; set; }
-        //public PoRParameters Por { get; set; }
     }
 
     public class StorageAvailability

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,146 +0,0 @@
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace CodexPlugin
-{
-    public interface IStoragePurchaseContract
-    {
-        string PurchaseId { get; }
-        StoragePurchaseRequest Purchase { get; }
-        ContentId ContentId { get; }
-        void WaitForStorageContractSubmitted();
-        void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished();
-    }
-
-    public class StoragePurchaseContract : IStoragePurchaseContract
-    {
-        private readonly ILog log;
-        private readonly CodexAccess codexAccess;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
-        private readonly DateTime contractPendingUtc = DateTime.UtcNow;
-        private DateTime? contractSubmittedUtc = DateTime.UtcNow;
-        private DateTime? contractStartedUtc;
-        private DateTime? contractFinishedUtc;
-
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            Purchase = purchase;
-
-            ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
-        }
-
-        public string PurchaseId { get; }
-        public StoragePurchaseRequest Purchase { get; }
-        public ContentId ContentId { get; }
-
-        public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
-        public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
-        public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
-
-        public void WaitForStorageContractSubmitted()
-        {
-            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
-            contractSubmittedUtc = DateTime.UtcNow;
-            LogSubmittedDuration();
-            AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
-        }
-
-        public void WaitForStorageContractStarted()
-        {
-            var timeout = Purchase.Expiry + gracePeriod;
-
-            WaitForStorageContractState(timeout, "started");
-            contractStartedUtc = DateTime.UtcNow;
-            LogStartedDuration();
-            AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartedUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
-            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-            contractFinishedUtc = DateTime.UtcNow;
-            LogFinishedDuration();
-            AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
-        }
-
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-
-            Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.State != lastState)
-                {
-                    lastState = purchaseStatus.State;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(sleep);
-
-                if (lastState == "errored")
-                {
-                    FrameworkAssert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
-                }
-            }
-        }
-
-        private void LogSubmittedDuration()
-        {
-            Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
-                $"( < {Time.FormatDuration(gracePeriod)})");
-        }
-
-        private void LogStartedDuration()
-        {
-            Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " +
-                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
-        }
-
-        private void LogFinishedDuration()
-        {
-            Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " +
-                $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})");
-        }
-
-        private void AssertDuration(TimeSpan? span, TimeSpan max, string message)
-        {
-            if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)");
-            if (span.Value.TotalDays >= max.TotalSeconds)
-            {
-                throw new Exception(nameof(MarketplaceAccess) +
-                    $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " +
-                    message);
-            }
-        }
-
-        private void Log(string msg)
-        {
-            log.Log($"[{PurchaseId}] {msg}");
-        }
-    }
-}

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -213,7 +213,8 @@ components:
           description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
         expiry:
           type: string
-          description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
+          description: Number as decimal string that represents expiry time of the request (in unix timestamp)
+
     StorageAsk:
       type: object
       required:
@@ -289,7 +290,6 @@ components:
           description: "Root hash of the content"
         originalBytes:
           type: integer
-          format: int64
           description: "Length of original content in bytes"
         blockSize:
           type: integer
@@ -304,18 +304,14 @@ components:
         totalBlocks:
           description: "Number of blocks stored by the node"
           type: integer
-          format: int64
         quotaMaxBytes:
           type: integer
-          format: int64
           description: "Maximum storage space used by the node"
         quotaUsedBytes:
           type: integer
-          format: int64
           description: "Amount of storage space currently in use"
         quotaReservedBytes:
           type: integer
-          format: int64
           description: "Amount of storage space reserved"
 
 servers:

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/EthAccount.cs

@@ -24,10 +24,5 @@ namespace GethPlugin
 
             return new EthAccount(ethAddress, account.PrivateKey);
         }
-
-        public override string ToString()
-        {
-            return EthAddress.ToString();
-        }
     }
 }

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -7,9 +7,9 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
+        public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
-            Pod = pod;
+            Containers = containers;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
@@ -17,9 +17,9 @@ namespace GethPlugin
             PubKey = pubKey;
         }
 
-        public RunningPod Pod { get; }
+        public RunningContainers Containers { get; }
         [JsonIgnore]
-        public RunningContainer Container {  get { return Pod.Containers.Single(); } }
+        public RunningContainer Container {  get { return Containers.Containers.Single(); } }
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -5,7 +5,6 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -28,7 +27,6 @@ namespace GethPlugin
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
         BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
-        BlockTimeEntry GetBlockForNumber(ulong number);
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -162,11 +160,6 @@ namespace GethPlugin
             return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
         }
 
-        public BlockTimeEntry GetBlockForNumber(ulong number)
-        {
-            return StartInteraction().GetBlockForNumber(number);
-        }
-
         protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,24 +6,24 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
         {
-            return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
+            return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,15 +31,15 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
             return starter.CollectMetricsFor(scrapeTargets);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
         {
-            runningPod = SerializeGate.Gate(runningPod);
-            return starter.CreateAccessForTarget(runningPod, target);
+            runningContainer = SerializeGate.Gate(runningContainer);
+            return starter.CreateAccessForTarget(runningContainer, target);
         }
 
         public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -1,5 +1,4 @@
 using Core;
-using IdentityModel;
 using KubernetesWorkflow.Types;
 using Logging;
 using System.Globalization;
@@ -178,41 +177,6 @@ namespace MetricsPlugin
         {
             return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
         }
-
-        public string AsCsv()
-        {
-            var allTimestamps = Sets.SelectMany(s => s.Values.Select(v => v.Timestamp)).Distinct().OrderDescending().ToArray();
-
-            var lines = new List<string>();
-            MakeLine(lines, e =>
-            {
-                e.Add("Metrics");
-                foreach (var ts in allTimestamps) e.Add(ts.ToEpochTime().ToString());
-            });
-
-            foreach (var set in Sets)
-            {
-                MakeLine(lines, e =>
-                {
-                    e.Add(set.Name);
-                    foreach (var ts in allTimestamps)
-                    {
-                        var value = set.Values.SingleOrDefault(v => v.Timestamp == ts);
-                        if (value == null) e.Add(" ");
-                        else e.Add(value.Value.ToString());
-                    }
-                });
-            }
-
-            return string.Join(Environment.NewLine, lines.ToArray());
-        }
-
-        private void MakeLine(List<string> lines, Action<List<string>> values)
-        {
-            var list = new List<string>();
-            values(list);
-            lines.Add(string.Join(",", list));
-        }
     }
 
     public class MetricsSet

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,7 +16,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -25,16 +25,16 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
+            var runningContainers = workflow.Start(1, recipe, startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
         {
-            var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
+            var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
         }
 

ProjectPlugins/WakuPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,34 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static RunningContainers[] DeployWakuNodes(this CoreInterface ci, int number, Action<IWakuSetup> setup)
+        {
+            return Plugin(ci).DeployWakuNodes(number, setup);
+        }
+
+        public static IWakuNode WrapWakuContainer(this CoreInterface ci, RunningContainer container)
+        {
+            return Plugin(ci).WrapWakuContainer(container);
+        }
+
+        public static IWakuNode StartWakuNode(this CoreInterface ci)
+        {
+            return ci.StartWakuNode(s => { });
+        }
+
+        public static IWakuNode StartWakuNode(this CoreInterface ci, Action<IWakuSetup> setup)
+        {
+            var rc = ci.DeployWakuNodes(1, setup);
+            return ci.WrapWakuContainer(rc.First().Containers.First());
+        }
+
+        private static WakuPlugin Plugin(CoreInterface ci)
+        {
+            return ci.GetPlugin<WakuPlugin>();
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuApiTypes.cs

@@ -0,0 +1,8 @@
+namespace WakuPlugin
+{
+    public class DebugInfoResponse
+    {
+        public string[] listenAddresses { get; set; } = Array.Empty<string>();
+        public string enrUri { get; set; } = string.Empty;
+    }
+}

ProjectPlugins/WakuPlugin/WakuContainerRecipe.cs

@@ -0,0 +1,38 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using Utils;
+
+namespace WakuPlugin
+{
+    public class WakuContainerRecipe : ContainerRecipeFactory
+    { 
+        public override string AppName => "waku";
+        //public override string Image => "statusteam/nim-waku:deploy-wakuv2-test";
+        public override string Image => "thatbenbierens/nim-waku:try";
+        public static string RestPortTag = "REST_PORT";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<WakuSetup>();
+
+            SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
+
+            AddEnvVar("WAKUNODE2_LOG_LEVEL", "TRACE");
+            AddEnvVar("WAKUNODE2_REST", "1");
+            AddExposedPortAndVar("WAKUNODE2_REST_PORT", RestPortTag);
+            AddEnvVar("WAKUNODE2_REST_ADDRESS", "0.0.0.0");
+
+            AddInternalPortAndVar("WAKUNODE2_TCP_PORT");
+            AddEnvVar("WAKUNODE2_RPC_ADDRESS", "0.0.0.0");
+
+            AddEnvVar("WAKUNODE2_DISCV5_DISCOVERY", "1");
+            AddInternalPortAndVar("WAKUNODE2_DISCV5_UDP_PORT");
+            AddEnvVar("WAKUNODE2_DISCV5_ENR_AUTO_UPDATEY", "1");
+
+            if (!string.IsNullOrEmpty(config.BootstrapEnr))
+            {
+                AddEnvVar("WAKUNODE2_DISCV5_BOOTSTRAP_NODE", config.BootstrapEnr);
+            }
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuNode.cs

@@ -0,0 +1,53 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public interface IWakuNode : IHasContainer
+    {
+        DebugInfoResponse DebugInfo();
+        void SubscribeToTopic(string topic);
+        void SendMessage(string topic, string message);
+        string[] GetMessages(string topic);
+    }
+
+    public class WakuNode : IWakuNode
+    {
+        private readonly IPluginTools tools;
+
+        public WakuNode(IPluginTools tools, RunningContainer container)
+        {
+            this.tools = tools;
+            Container = container;
+        }
+
+        public RunningContainer Container { get; }
+
+        public DebugInfoResponse DebugInfo()
+        {
+            return Api().HttpGetJson<DebugInfoResponse>("debug/v1/info");
+        }
+
+        public void SubscribeToTopic(string topic)
+        {
+            var response = Api().HttpPostString<string>(route: "relay/v1/subscriptions", body: topic);
+        }
+
+        public void SendMessage(string topic, string message)
+        {
+            var response = Api().HttpPostString<string>($"relay/v1/messages/{topic}", message);
+        }
+
+        public string[] GetMessages(string topic)
+        {
+            var response = Api().HttpGetString($"relay/v1/messages/{topic}");
+            return new[] { "" };
+        }
+
+        private IEndpoint Api()
+        {
+            var address = Container.GetAddress(tools.GetLog(), WakuContainerRecipe.RestPortTag);
+            return tools.CreateHttp().CreateEndpoint(address, "", logAlias: "waku");
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuPlugin.cs

@@ -0,0 +1,71 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public class WakuPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
+    {
+        private readonly IPluginTools tools;
+        private readonly WakuStarter starter;
+
+        public WakuPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+            starter = new WakuStarter(tools);
+        }
+
+        public string LogPrefix => "(Waku) ";
+
+        public void Announce()
+        {
+            tools.GetLog().Log($"Loaded with Waku plugin.");
+        }
+
+        public void AddMetadata(IAddMetadata metadata)
+        {
+            //metadata.Add("codexid", codexStarter.GetCodexId());
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public RunningContainers[] DeployWakuNodes(int numberOfNodes, Action<IWakuSetup> setup)
+        {
+            return starter.Start(numberOfNodes, setup);
+        }
+
+        public IWakuNode WrapWakuContainer(RunningContainer container)
+        {
+            container = SerializeGate.Gate(container);
+            return starter.Wrap(container);
+        }
+
+        //public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        //{
+        //    containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
+        //    return codexStarter.WrapCodexContainers(coreInterface, containers);
+        //}
+
+        //public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
+        //{
+        //    var codexSetup = GetSetup(1, setup);
+        //    if (codexSetup.MarketplaceConfig == null) return;
+
+        //    var mconfig = codexSetup.MarketplaceConfig;
+        //    foreach (var node in result)
+        //    {
+        //        mconfig.GethNode.SendEth(node, mconfig.InitialEth);
+        //        mconfig.CodexContracts.MintTestTokens(mconfig.GethNode, node, mconfig.InitialTokens);
+        //    }
+        //}
+
+        //private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
+        //{
+        //    var codexSetup = new CodexSetup(numberOfNodes);
+        //    codexSetup.LogLevel = defaultLogLevel;
+        //    setup(codexSetup);
+        //    return codexSetup;
+        //}
+    }
+}

ProjectPlugins/WakuPlugin/WakuPlugin.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/WakuPlugin/WakuSetup.cs

@@ -0,0 +1,26 @@
+namespace WakuPlugin
+{
+    public interface IWakuSetup
+    {
+        IWakuSetup WithName(string name);
+        IWakuSetup WithBootstrapNode(IWakuNode node);
+    }
+
+    public class WakuSetup : IWakuSetup
+    {
+        internal string? Name { get; private set; }
+        internal string? BootstrapEnr { get; private set; }
+
+        public IWakuSetup WithName(string name)
+        {
+            Name = name;
+            return this;
+        }
+
+        public IWakuSetup WithBootstrapNode(IWakuNode node)
+        {
+            BootstrapEnr = node.DebugInfo().enrUri;
+            return this;
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuStarter.cs

@@ -0,0 +1,45 @@
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public class WakuStarter
+    {
+        private readonly IPluginTools tools;
+
+        public WakuStarter(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public RunningContainers[] Start(int numberOfNodes, Action<IWakuSetup> setup)
+        {
+            var result = new List<RunningContainers>();
+            var workflow = tools.CreateWorkflow();
+            var startupConfig = CreateStartupConfig(setup);
+
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(workflow.Start(1, new WakuContainerRecipe(), startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        public IWakuNode Wrap(RunningContainer container)
+        {
+            return new WakuNode(tools, container);
+        }
+
+        private StartupConfig CreateStartupConfig(Action<IWakuSetup> setup)
+        {
+            var config = new WakuSetup();
+            setup(config);
+            var startupConfig = new StartupConfig();
+            startupConfig.Add(config);
+            startupConfig.NameOverride = config.Name;
+            return startupConfig;
+        }
+    }
+}

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -148,7 +148,7 @@ namespace ContinuousTests
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
-            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
+            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
         }
 
         private void PerformCleanup(ILog log)
@@ -157,7 +157,7 @@ namespace ContinuousTests
             log.Log("Cleaning up test namespace...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
-            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
+            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
             log.Log("Cleanup finished.");
         }
     }

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -49,8 +49,8 @@ namespace ContinuousTests
             var start = startUtc.ToString("o");
             var end = endUtc.ToString("o");
 
-            var containerName = container.RunningPod.StartResult.Deployment.Name;
-            var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
+            var containerName = container.RunningContainers.StartResult.Deployment.Name;
+            var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
 
             //container_name : codex3-5 - deploymentName as stored in pod
             // pod_namespace : codex - continuous - nolimits - tests - 1

Tests/CodexContinuousTests/NodeRunner.cs

@@ -64,7 +64,7 @@ namespace ContinuousTests
             }
             finally
             {
-                entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false);
+                entryPoint.Tools.CreateWorkflow().DeleteNamespace();
             }
         }
 

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -54,8 +54,7 @@ namespace ContinuousTests
 
                     entryPoint.Decommission(
                         deleteKubernetesResources: false, // This would delete the continuous test net.
-                        deleteTrackedFiles: true,
-                        waitTillDone: false
+                        deleteTrackedFiles: true
                     );
                     runFinishedHandle.Set();
                 }
@@ -126,8 +125,8 @@ namespace ContinuousTests
             foreach (var node in nodes)
             {
                 var container = node.Container;
-                var deploymentName = container.RunningPod.StartResult.Deployment.Name;
-                var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
+                var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
+                var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
                 elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
@@ -296,13 +295,13 @@ namespace ContinuousTests
             return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
         }
 
-        private RunningPod[] SelectRandomContainers()
+        private RunningContainers[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
+            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
             if (number == -1) return containers.ToArray();
 
-            var result = new RunningPod[number];
+            var result = new RunningContainers[number];
             for (var i = 0; i < number; i++)
             {
                 result[i] = containers.PickOneRandom();

Tests/CodexContinuousTests/StartupChecker.cs

@@ -43,13 +43,13 @@ namespace ContinuousTests
             var workflow = entryPoint.Tools.CreateWorkflow();
             foreach (var instance in deployment.CodexInstances)
             {
-                foreach (var container in instance.Pod.Containers)
+                foreach (var container in instance.Containers.Containers)
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
                     log.Log(
-                        $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
+                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
@@ -92,7 +92,7 @@ namespace ContinuousTests
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
             var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,4 +1,3 @@
-using CodexPlugin;
 using CodexTests;
 using DistTestCore;
 using FileUtils;
@@ -8,45 +7,35 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class DownloadTests : AutoBootstrapDistTest
+    public class DownloadTests : CodexDistTest
     {
-        [Test]
-        [Combinatorial]
+        [TestCase(3, 500)]
+        [TestCase(5, 100)]
+        [TestCase(10, 256)]
         [UseLongTimeouts]
-        public void ParallelDownload(
-            [Values(1, 3, 5)] int numberOfFiles,
-            [Values(10, 50, 100)] int filesizeMb)
+        public void ParallelDownload(int numberOfNodes, int filesizeMb)
         {
-            var host = StartCodex();
-            var client = StartCodex();
+            var group = AddCodex(numberOfNodes);
+            var host = AddCodex();
 
-            var testfiles = new List<TrackedFile>();
-            var contentIds = new List<ContentId>();
-            var downloadedFiles = new List<TrackedFile?>();
-
-            for (int i = 0; i < numberOfFiles; i++)
+            foreach (var node in group)
             {
-                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
-                contentIds.Add(new ContentId());
-                downloadedFiles.Add(null);
+                host.ConnectToPeer(node);
             }
 
-            for (int i = 0; i < numberOfFiles; i++)
+            var testFile = GenerateTestFile(filesizeMb.MB());
+            var contentId = host.UploadFile(testFile);
+            var list = new List<Task<TrackedFile?>>();
+
+            foreach (var node in group)
             {
-                contentIds[i] = host.UploadFile(testfiles[i]);
+                list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));
             }
 
-            var downloadTasks = new List<Task>();
-            for (int i = 0; i < numberOfFiles; i++)
+            Task.WaitAll(list.ToArray());
+            foreach (var task in list)
             {
-                downloadTasks.Add(Task.Run(() => { downloadedFiles[i] = client.DownloadContent(contentIds[i]); }));
-            }
-
-            Task.WaitAll(downloadTasks.ToArray());
-
-            for (int i = 0; i < numberOfFiles; i++)
-            {
-                testfiles[i].AssertIsEqual(downloadedFiles[i]);
+                testFile.AssertIsEqual(task.Result);
             }
         }
     }

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -48,7 +48,7 @@ namespace CodexLongTests.BasicTests
 
             var expectedFile = GenerateTestFile(sizeMB);
 
-            var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
+            var node = AddCodex(s => s.WithStorageQuota((size + 10).MB()));
 
             var uploadStart = DateTime.UtcNow;
             var cid = node.UploadFile(expectedFile);

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -6,12 +6,10 @@ namespace CodexLongTests.BasicTests
 {
     public class TestInfraTests : CodexDistTest
     {
-        [Test]
-        [UseLongTimeouts]
-        [Ignore("Not supported atm")]
+        [Test, UseLongTimeouts]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = StartCodex(1000, s => s.EnableMetrics());
+            var group = AddCodex(1000, s => s.EnableMetrics());
 
             var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray();
 
@@ -19,14 +17,12 @@ namespace CodexLongTests.BasicTests
                 "Not all created nodes provided a unique id.");
         }
 
-        [Test]
-        [UseLongTimeouts]
-        [Ignore("Not supported atm")]
+        [Test, UseLongTimeouts]
         public void TestInfraSupportsManyConcurrentPods()
         {
             for (var i = 0; i < 20; i++)
             {
-                var n = StartCodex();
+                var n = AddCodex();
 
                 Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id));
             }

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -8,39 +8,41 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class UploadTests : AutoBootstrapDistTest
+    public class UploadTests : CodexDistTest
     {
-        [Test]
-        [Combinatorial] 
+        [TestCase(3, 50)]
+        [TestCase(5, 75)]
+        [TestCase(10, 25)]
         [UseLongTimeouts]
-        public void ParallelUpload(
-            [Values(1, 3, 5)] int numberOfFiles,
-            [Values(10, 50, 100)] int filesizeMb)
+        public void ParallelUpload(int numberOfNodes, int filesizeMb)
         {
-            var host = StartCodex();
-            var client = StartCodex();
+            var group = AddCodex(numberOfNodes);
+            var host = AddCodex();
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
 
             var testfiles = new List<TrackedFile>();
-            var contentIds = new List<ContentId>();
+            var contentIds = new List<Task<ContentId>>();
 
-            for (int i = 0; i < numberOfFiles; i++)
+            for (int i = 0; i < group.Count(); i++)
             {
                 testfiles.Add(GenerateTestFile(filesizeMb.MB()));
-                contentIds.Add(new ContentId());
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
             }
-
-            var uploadTasks = new List<Task>();
-            for (int i = 0; i < numberOfFiles; i++)
+            var downloads = new List<Task<TrackedFile?>>();
+            for (int i = 0; i < group.Count(); i++)
             {
-                uploadTasks.Add(Task.Run(() => { contentIds[i] = host.UploadFile(testfiles[i]); }));
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); }));
             }
-
-            Task.WaitAll(uploadTasks.ToArray());
-
-            for (int i = 0; i < numberOfFiles; i++)
+            Task.WaitAll(downloads.ToArray());
+            for (int i = 0; i < group.Count(); i++)
             {
-                var downloaded = client.DownloadContent(contentIds[i]);
-                testfiles[i].AssertIsEqual(downloaded);
+                testfiles[i].AssertIsEqual(downloads[i].Result);
             }
         }
     }

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -15,9 +15,9 @@ namespace CodexLongTests.DownloadConnectivityTests
             [Values(10, 15, 20)] int numberOfNodes,
             [Values(10, 100)] int sizeMBs)
         {
-            var nodes = StartCodex(numberOfNodes);
+            for (var i = 0; i < numberOfNodes; i++) AddCodex();
 
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -1,120 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.ScalabilityTests
-{
-    [TestFixture]
-    public class MultiPeerDownloadTests : AutoBootstrapDistTest
-    {
-        [Test]
-        [DontDownloadLogs]
-        [UseLongTimeouts]
-        [Combinatorial]
-        public void MultiPeerDownload(
-            [Values(5, 10, 20)] int numberOfHosts,
-            [Values(100, 1000)] int fileSize
-        )
-        {
-            var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
-            var file = GenerateTestFile(fileSize.MB());
-            var cid = hosts[0].UploadFile(file);
-            var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
-
-            var uploadLog = Ci.DownloadLog(hosts[0]);
-            var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
-            var blockCids = uploadLog
-                .FindLinesThatContain("Putting block into network store")
-                .Select(s =>
-                {
-                    var start = s.IndexOf("cid=") + 4;
-                    var end = s.IndexOf(" count=");
-                    var len = end - start;
-                    return s.Substring(start, len);
-                })
-                .ToArray();
-
-            Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
-
-            foreach (var h in hosts) h.DownloadContent(cid);
-
-            var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
-            var resultFile = client.DownloadContent(cid);
-            resultFile!.AssertIsEqual(file);
-
-            var downloadLog = Ci.DownloadLog(client);
-            var host = string.Empty;
-            var blockCidHostMap = new Dictionary<string, string>();
-            downloadLog.IterateLines(line =>
-            {
-                if (line.Contains("peer=") && line.Contains(" len="))
-                {
-                    var start = line.IndexOf("peer=") + 5;
-                    var end = line.IndexOf(" len=");
-                    var len = end - start;
-                    host = line.Substring(start, len);
-                }
-                else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
-                {
-                    var start = line.IndexOf("cid=") + 4;
-                    var end = line.IndexOf(" count=");
-                    var len = end - start;
-                    var blockCid = line.Substring(start, len);
-
-                    blockCidHostMap.Add(blockCid, host);
-                    host = string.Empty;
-                }
-            });
-
-            var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
-            //PrintFullMap(blockCidHostMap);
-            PrintOverview(blockCidHostMap);
-
-            Log("Expected number of blocks: " + expectedNumberOfBlocks);
-            Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
-            Log("Total blocks fetched by hosts: " + totalFetched);
-            Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
-        }
-
-        private void PrintOverview(Dictionary<string, string> blockCidHostMap)
-        {
-            var overview = new Dictionary<string, int>();
-            foreach (var pair in blockCidHostMap)
-            {
-                if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
-                else overview[pair.Value]++;
-            }
-
-            Log("Blocks fetched per host:");
-            foreach (var pair in overview)
-            {
-                Log($"Host: {pair.Key} = {pair.Value}");
-            }
-        }
-
-        private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
-        {
-            Log("Per block, host it was fetched from:");
-            foreach (var pair in blockCidHostMap)
-            {
-                if (string.IsNullOrEmpty(pair.Value))
-                {
-                    Log($"block: {pair.Key} = Not seen");
-                }
-                else
-                {
-                    Log($"block: {pair.Key} = '{pair.Value}'");
-                }
-            }
-        }
-
-        private long RoundUp(long filesize, long blockSize)
-        {
-            double f = filesize;
-            double b = blockSize;
-
-            var result = Math.Ceiling(f / b);
-            return Convert.ToInt64(result);
-        }
-    }
-}