Merge branch 'master' into feature/waku-plugin

# Conflicts:
#	Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
#	cs-codex-dist-testing.sln

Framework/ArgsUniform/ArgsUniform.cs

@@ -4,8 +4,9 @@ namespace ArgsUniform
 {
     public class ArgsUniform<T>
     {
-        private readonly Assigner<T> assigner;
         private readonly Action printAppInfo;
+        private readonly object? defaultsProvider;
+        private readonly IEnv.IEnv env;
         private readonly string[] args;
         private const int cliStart = 8;
         private const int shortStart = 38;
@@ -30,9 +31,9 @@ namespace ArgsUniform
         public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
         {
             this.printAppInfo = printAppInfo;
+            this.defaultsProvider = defaultsProvider;
+            this.env = env;
             this.args = args;
-
-            assigner = new Assigner<T>(env, args, defaultsProvider);
         }
 
         public T Parse(bool printResult = false)
@@ -41,7 +42,7 @@ namespace ArgsUniform
             {
                 printAppInfo();
                 PrintHelp();
-                Environment.Exit(0);
+                throw new Exception();
             }
 
             var result = Activator.CreateInstance<T>();
@@ -52,16 +53,18 @@ namespace ArgsUniform
                 var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
                 if (attr != null)
                 {
-                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
                     {
-                        missingRequired.Add(uniformProperty);
+                        {
+                            missingRequired.Add(uniformProperty);
+                        }
                     }
                 }
             }
 
             if (missingRequired.Any())
             {
-                PrintResults(printResult,result, uniformProperties);
+                PrintResults(result, uniformProperties);
                 Print("");
                 foreach (var missing in missingRequired)
                 {
@@ -72,39 +75,37 @@ namespace ArgsUniform
                 }
 
                 PrintHelp();
-                Environment.Exit(1);
+                throw new ArgumentException("Unable to assemble all required arguments");
             }
 
-            PrintResults(printResult, result, uniformProperties);
+            if (printResult)
+            {
+                PrintResults(result, uniformProperties);
+            }
 
             return result;
         }
 
+        private void PrintResults(T result, PropertyInfo[] uniformProperties)
+        {
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
         public void PrintHelp()
         {
             Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
-            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
-            foreach (var prop in props)
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
+            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
+            foreach (var attr in attrs)
             {
-                var a = prop.GetCustomAttribute<UniformAttribute>();
-                if (a != null)
-                {
-                    var optional = !a.Required ? " (optional)" : "";
-                    var def = assigner.DescribeDefaultFor(prop);
-                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
-                }
-            }
-            Print("");
-        }
-
-        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
-        {
-            if (!printResult) return;
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
+                var a = attr!;
+                var optional = !a.Required ? " *" : "";
+                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
             }
             Print("");
         }
@@ -114,7 +115,7 @@ namespace ArgsUniform
             Console.WriteLine(msg);
         }
 
-        private void PrintAligned(string cli, string s, string env, string desc, string def)
+        private void PrintAligned(string cli, string s, string env, string desc)
         {
             Console.CursorLeft = cliStart;
             Console.Write(cli);
@@ -123,8 +124,132 @@ namespace ArgsUniform
             Console.CursorLeft = envStart;
             Console.Write(env);
             Console.CursorLeft = descStart;
-            Console.Write(desc + " ");
-            Console.Write(def + Environment.NewLine);
+            Console.Write(desc + Environment.NewLine);
+        }
+
+        private object GetDefaultValue(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+
+        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var currentValue = uniformProperty.GetValue(result);
+            var isEmptryString = (currentValue as string) == string.Empty;
+            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
+
+            if (defaultsProvider == null) return false;
+
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return false;
+
+            var value = defaultProperty.GetValue(defaultsProvider);
+            if (value != null)
+            {
+                return Assign(result, uniformProperty, value);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    
+                    throw new NotSupportedException();
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
         }
     }
 }

Framework/ArgsUniform/Assigner.cs

@@ -1,160 +0,0 @@
-using System.Reflection;
-
-namespace ArgsUniform
-{
-    public class Assigner<T>
-    {
-        private readonly IEnv.IEnv env;
-        private readonly string[] args;
-        private readonly object? defaultsProvider;
-
-        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
-        {
-            this.env = env;
-            this.args = args;
-            this.defaultsProvider = defaultsProvider;
-        }
-
-        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        public string DescribeDefaultFor(PropertyInfo property)
-        {
-            var obj = Activator.CreateInstance<T>();
-            var defaultValue = GetDefaultValue(obj, property);
-            if (defaultValue == null) return "";
-            if (defaultValue is string str)
-            {
-                return "\"" + str + "\"";
-            }
-            return defaultValue.ToString() ?? string.Empty;
-        }
-
-        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
-        {
-            // Get value from object's static initializer if it's there.
-            var currentValue = uniformProperty.GetValue(result);
-            if (currentValue != null) return currentValue;
-
-            // Get value from defaults-provider object if it's there.
-            if (defaultsProvider == null) return null;
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return null;
-            return defaultProperty.GetValue(defaultsProvider);
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var defaultValue = GetDefaultValue(result, uniformProperty);
-            var isEmptryString = (defaultValue as string) == string.Empty;
-            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
-            {
-                return Assign(result, uniformProperty, defaultValue);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-
-                    throw new NotSupportedException();
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
-        }
-
-        private static object GetDefaultValueForType(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-    }
-}

Framework/Core/DownloadedLog.cs

@@ -4,7 +4,6 @@ namespace Core
 {
     public interface IDownloadedLog
     {
-        void IterateLines(Action<string> action);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
@@ -19,19 +18,6 @@ namespace Core
             this.logFile = logFile;
         }
 
-        public void IterateLines(Action<string> action)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                action(line);
-                line = streamReader.ReadLine();
-            }
-        }
-
         public string[] GetLinesContaining(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);

Framework/Core/Http.cs

@@ -58,7 +58,7 @@ namespace Core
         {
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
+                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
             }
         }
 

Framework/Core/TimeSet.cs

@@ -2,31 +2,10 @@
 {
     public interface ITimeSet
     {
-        /// <summary>
-        /// Timeout for a single HTTP call.
-        /// </summary>
         TimeSpan HttpCallTimeout();
-
-        /// <summary>
-        /// Maximum total time to attempt to make a successful HTTP call to a service.
-        /// When HTTP calls time out during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan HttpRetryTimeout();
-
-        /// <summary>
-        /// After a failed HTTP call, wait this long before trying again.
-        /// </summary>
+        int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
+        TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
     }
 
@@ -37,9 +16,9 @@
             return TimeSpan.FromMinutes(3);
         }
 
-        public TimeSpan HttpRetryTimeout()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromMinutes(10);
+            return 3;
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -47,7 +26,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan K8sOperationRetryDelay()
+        public TimeSpan WaitForK8sServiceDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -62,27 +41,27 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(30);
+            return TimeSpan.FromHours(2);
         }
 
-        public TimeSpan HttpRetryTimeout()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromHours(2.2);
+            return 1;
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(20);
+            return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan K8sOperationRetryDelay()
+        public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(30);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromHours(1);
+            return TimeSpan.FromMinutes(15);
         }
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -16,7 +16,7 @@
     public class MarketAverage
     {
         public int NumberOfFinished { get; set; }
-        public int TimeRangeSeconds { get; set; }
+        public TimeSpan TimeRange { get; set; }
         public float Price { get; set; }
         public float Size { get; set; }
         public float Duration { get; set; }

Framework/FileUtils/TrackedFile.cs

@@ -65,7 +65,7 @@ namespace FileUtils
 
                 if (readExpected == 0 && readActual == 0)
                 {
-                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
+                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
                     return;
                 }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,11 +43,6 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
-        public void WaitUntilOnline(RunningContainer container)
-        {
-            WaitUntilDeploymentOnline(container.Recipe.Name);
-        }
-
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -377,6 +372,7 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
+            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -705,14 +701,14 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxTimeout: TimeSpan.FromMinutes(1),
+                maxRetries: 6,
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -868,7 +864,7 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
         }
 
         private void WaitUntilDeploymentOnline(string deploymentName)
@@ -877,7 +873,7 @@ namespace KubernetesWorkflow
             {
                 var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            }, nameof(WaitUntilDeploymentOnline));
+            });
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -887,7 +883,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            }, nameof(WaitUntilDeploymentOffline));
+            });
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -896,19 +892,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            }, nameof(WaitUntilPodsForDeploymentAreOffline));
+            });
         }
 
-        private void WaitUntil(Func<bool> predicate, string msg)
+        private void WaitUntil(Func<bool> predicate)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
             }
             finally
             {
-                sw.End(msg, 1);
+                sw.End("", 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningPod runningPod);
-        void OnContainersStopped(RunningPod runningPod);
+        void OnContainersStarted(RunningContainers runningContainers);
+        void OnContainersStopped(RunningContainers runningContainers);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningPod runningPod)
+        public void OnContainersStarted(RunningContainers runningContainers)
         {
         }
 
-        public void OnContainersStopped(RunningPod runningPod)
+        public void OnContainersStopped(RunningContainers runningContainers)
         {
         }
 

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -73,6 +73,13 @@ namespace KubernetesWorkflow.Recipe
             return p;
         }
 
+        protected Port AddInternalPort(int number, string tag = "", PortProtocol protocol = PortProtocol.TCP)
+        {
+            var p = factory.CreateInternalPort(number, tag, protocol);
+            internalPorts.Add(p);
+            return p;
+        }
+
         protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
             AddEnvVar(name, AddExposedPort(tag, protocol));

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -16,7 +16,12 @@ namespace KubernetesWorkflow.Recipe
 
         public Port CreateInternalPort(string tag, PortProtocol protocol)
         {
-            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+            return CreateInternalPort(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateInternalPort(int number, string tag, PortProtocol protocol)
+        {
+            return new Port(number, tag, protocol);
         }
 
         public Port CreateExternalPort(int number, string tag, PortProtocol protocol)

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,12 +9,12 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningPod pod);
+        PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningPod pod, bool waitTillStopped);
+        void Stop(RunningContainers containers, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,36 +60,25 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return new FutureContainers(rc, this);
-            });
-        }
-
-        public void WaitUntilOnline(RunningPod rc)
-        {
-            K8s(controller =>
-            {
-                foreach (var c in rc.Containers)
-                {
-                    controller.WaitUntilOnline(c);
-                }
+                return rc;
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningPod pod)
+        public PodInfo GetPodInfo(RunningContainers containers)
         {
-            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -97,12 +86,12 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningPod runningPod, bool waitTillStopped)
+        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
+                controller.Stop(runningContainers.StartResult, waitTillStopped);
+                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -1,20 +0,0 @@
-namespace KubernetesWorkflow.Types
-{
-    public class FutureContainers
-    {
-        private readonly RunningPod runningPod;
-        private readonly StartupWorkflow workflow;
-
-        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
-        {
-            this.runningPod = runningPod;
-            this.workflow = workflow;
-        }
-
-        public RunningPod WaitForOnline()
-        {
-            workflow.WaitUntilOnline(runningPod);
-            return runningPod;
-        }
-    }
-}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
         public ContainerAddress[] Addresses { get; }
 
         [JsonIgnore]
-        public RunningPod RunningPod { get; internal set; } = null!;
+        public RunningContainers RunningContainers { get; internal set; } = null!;
 
         public Address GetAddress(ILog log, string portTag)
         {

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -2,15 +2,15 @@
 
 namespace KubernetesWorkflow.Types
 {
-    public class RunningPod
+    public class RunningContainers
     {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
 
-            foreach (var c in containers) c.RunningPod = this;
+            foreach (var c in containers) c.RunningContainers = this;
         }
 
         public StartupConfig StartupConfig { get; }
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public string Name
         {
-            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
+            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
         }
 
         public string Describe()
@@ -31,7 +31,12 @@ namespace KubernetesWorkflow.Types
 
     public static class RunningContainersExtensions
     {
-        public static string Describe(this RunningPod[] runningContainers)
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
         {
             return string.Join(",", runningContainers.Select(c => c.Describe()));
         }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,14 +18,6 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
-        public BlockTimeEntry Get(ulong blockNumber)
-        {
-            bounds.Initialize();
-            var b = cache.Get(blockNumber);
-            if (b != null) return b;
-            return GetBlock(blockNumber);
-        }
-
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -121,12 +121,5 @@ namespace NethereumWorkflow
                 to: toBlock.Value
             );
         }
-
-        public BlockTimeEntry GetBlockForNumber(ulong number)
-        {
-            var wrapper = new Web3Wrapper(web3, log);
-            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
-            return blockTimeFinder.Get(number);
-        }
     }
 }

Framework/Utils/Formatter.cs

@@ -1,6 +1,4 @@
-using System.Globalization;
-
-namespace Utils
+namespace Utils
 {
     public static class Formatter
     {
@@ -12,7 +10,7 @@ namespace Utils
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
+            return digit.ToString() + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,7 +2,6 @@
 {
     public class NumberSource
     {
-        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -12,12 +11,8 @@
 
         public int GetNextNumber()
         {
-            var n = -1;
-            lock (@lock)
-            {
-                n = number;
-                number++;
-            }
+            var n = number;
+            number++;
             return n;
         }
     }

Framework/Utils/Time.cs

@@ -1,6 +1,4 @@
-using System.Diagnostics;
-
-namespace Utils
+namespace Utils
 {
     public static class Time
     {
@@ -59,66 +57,60 @@ namespace Utils
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate, string msg)
+        public static void WaitUntil(Func<bool> predicate)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
         {
             var start = DateTime.UtcNow;
-            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                var duration = DateTime.UtcNow - start;
-                if (duration > timeout)
+                if (DateTime.UtcNow - start > timeout)
                 {
-                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
+                    throw new TimeoutException("Operation timed out.");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
-                tries++;
             }
         }
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, TimeSpan.FromSeconds(30), description);
+            Retry(action, 1, description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, TimeSpan.FromSeconds(30), description);
+            return Retry(action, 1, description);
         }
 
-        public static void Retry(Action action, TimeSpan maxTimeout, string description)
+        public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
+        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var tries = 1;
-            var tryInfo = new List<(Exception, TimeSpan)>();
-
+            var retries = 0;
+            var exceptions = new List<Exception>();
             while (true)
             {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
+                if (retries > maxRetries)
                 {
-                    var info = FormatTryInfos(tryInfo);
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
-                var sw = Stopwatch.StartNew();
                 try
                 {
                     action();
@@ -126,42 +118,25 @@ namespace Utils
                 }
                 catch (Exception ex)
                 {
-                    tryInfo.Add((ex, sw.Elapsed));
-                    tries++;
+                    exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
-        {
-            return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
-        }
-
-        private static string FormatTryInfo((Exception, TimeSpan) info, int index)
-        {
-            return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
-        }
-
-        private static Action<int> failedCallback = i => { };
-        public static void SetRetryFailedCallback(Action<int> onRetryFailed)
-        {
-            failedCallback = onRetryFailed;
-        }
-
-        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var tries = 1;
+            var retries = 0;
             var exceptions = new List<Exception>();
-
             while (true)
             {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -171,8 +146,7 @@ namespace Utils
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
-                    failedCallback(tries);
-                    tries++;
+                    retries++;
                 }
 
                 Sleep(retryTime);

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -5,9 +5,6 @@ using Nethereum.ABI;
 using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -31,7 +28,6 @@ namespace CodexContractsPlugin
         SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
     }
 
-    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -90,7 +86,7 @@ namespace CodexContractsPlugin
                     {
                         var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
                         var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
                         result.RequestId = e.Event.RequestId;
                         return result;
                     })
@@ -103,7 +99,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
                 return result;
             }).ToArray();
         }
@@ -114,7 +110,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
                 return result;
             }).ToArray();
         }
@@ -125,7 +121,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
                 result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
                 return result;
             }).ToArray();
@@ -137,7 +133,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
                 return result;
             }).ToArray();
         }
@@ -170,11 +166,6 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
-        private BlockTimeEntry GetBlock(ulong number)
-        {
-            return gethNode.GetBlockForNumber(number);
-        }
-
         private EthAddress GetEthAddressFromTransaction(string transactionHash)
         {
             var transaction = gethNode.GetTransaction(transactionHash);

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,7 +59,7 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            }, nameof(DeployContract));
+            });
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@@ -71,7 +71,7 @@ namespace CodexContractsPlugin
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
 
             Log("Synced. Codex SmartContracts deployed.");
 
@@ -83,9 +83,9 @@ namespace CodexContractsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate, string msg)
+        private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,5 +1,4 @@
-using CodexContractsPlugin.Marketplace;
-using KubernetesWorkflow;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -54,18 +53,7 @@ namespace CodexContractsPlugin
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            var byteCode = artifact["bytecode"];
-            var abiResult = abi!.ToString(Formatting.None);
-            var byteCodeResult = byteCode!.ToString(Formatting.None);
-
-            if (byteCodeResult
-                .ToLowerInvariant()
-                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
-            {
-                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
-            }
-
-            return abiResult;
+            return abi!.ToString(Formatting.None);
         }
 
         private static string Retry(Func<string> fetch)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,6 +1,5 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
@@ -8,7 +7,7 @@ namespace CodexContractsPlugin.Marketplace
     public partial class Request : RequestBase
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
@@ -17,26 +16,26 @@ namespace CodexContractsPlugin.Marketplace
     public partial class RequestFulfilledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 
     public partial class RequestCancelledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 
     public partial class SlotFilledEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
         public EthAddress Host { get; set; }
     }
 
     public partial class SlotFreedEventDTO
     {
         [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
+        public ulong BlockNumber { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -1,14 +1 @@
 This code was generated using the Nethereum code generator, here: http://playground.nethereum.com
-
-1. Go to site -> Abi Code Gen.
-1. Contract name = "Marketplace".
-1. In container, get "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json".
-1. Save only ABI section as new JSON. (top-level is a json array.)
-1. From original JSON get byte code.
-1. Put ABI JSON and byte code into site.
-1. Generate.
-1. From site generated code, copy `public partial class MarketplaceDeployment` and everything after it. (be considerate of namespace brackets!)
-1. In Marketplace/Marketplace.cs, replace content of 'namespace CodexContractsPlugin.Marketplace'.
-
-
-

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -29,31 +29,31 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningPod Deploy(DiscordBotStartupConfig config)
+        public RunningContainers Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
+        public RunningContainers DeployRewarder(RewarderBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartRewarderContainer(workflow, config);
         }
 
-        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
         }
 
-        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.Add(config);
-            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
+            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig);
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
-        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
         {
             return Plugin(ci).DeployRewarder(config);
         }

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "0F-C8-02-1E-2C-2C-15-F6-91-6A-01-31-11-49-95-06-79-26-25-BF-27-3C-A8-2E-5F-7F-34-FD-C0-57-A0-9A";
+        private const string OpenApiYamlHash = "5A-B0-2A-AC-42-B1-A2-49-6F-9D-4E-D8-56-40-10-A6-67-F4-0D-2A-9F-E0-84-5C-EB-B8-2D-4F-D8-56-79-6C";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -38,7 +38,7 @@ namespace CodexPlugin
             if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
         }
 
-        public void CheckCompatibility(RunningPod[] containers)
+        public void CheckCompatibility(RunningContainers[] containers)
         {
             if (checkPassed) return;
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -13,7 +13,7 @@ namespace CodexPlugin
         private readonly Mapper mapper = new Mapper();
         private bool hasContainerCrashed;
 
-        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
+        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
             Container = container;
@@ -23,7 +23,7 @@ namespace CodexPlugin
             CrashWatcher.Start(this);
         }
 
-        public RunningPod Container { get; }
+        public RunningContainer Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
         public DebugInfo GetDebugInfo()
@@ -136,7 +136,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-267266a-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1524803-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
     public class CodexDeployment
     {
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
-            CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
-            RunningPod? discordBotContainer, DeploymentMetadata metadata,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
             String id)
         {
             Id = id;
@@ -24,20 +24,20 @@ namespace CodexPlugin
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningPod? PrometheusContainer { get; }
-        public RunningPod? DiscordBotContainer { get; }
+        public RunningContainers? PrometheusContainer { get; }
+        public RunningContainers? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningPod pod, DebugInfo info)
+        public CodexInstance(RunningContainers containers, DebugInfo info)
         {
-            Pod = pod;
+            Containers = containers;
             Info = info;
         }
 
-        public RunningPod Pod { get; }
+        public RunningContainers Containers { get; }
         public DebugInfo Info { get; }
     }
 

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -1,105 +0,0 @@
-using System.Globalization;
-
-namespace CodexPlugin
-{
-    public class CodexLogLine
-    {
-        public static CodexLogLine? Parse(string line)
-        {
-            try
-            {
-                if (string.IsNullOrEmpty(line) ||
-                    line.Length < 34 ||
-                    line[3] != ' ' ||
-                    line[33] != ' ') return null;
-
-                line = line.Replace(Environment.NewLine, string.Empty);
-
-                var level = line.Substring(0, 3);
-                var dtLine = line.Substring(4, 23);
-
-                var firstEqualSign = line.IndexOf('=');
-                var msgStart = 34;
-                var msgEnd = line.Substring(0, firstEqualSign).LastIndexOf(' ');
-                var msg = line.Substring(msgStart, msgEnd - msgStart).Trim();
-                var attrsLine = line.Substring(msgEnd);
-
-                var attrs = SplitAttrs(attrsLine);
-
-                var format = "yyyy-MM-dd HH:mm:ss.fff";
-                var dt = DateTime.ParseExact(dtLine, format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime();
-
-                return new CodexLogLine()
-                {
-                    LogLevel = level,
-                    TimestampUtc = dt,
-                    Message = msg,
-                    Attributes = attrs
-                };
-            }
-            catch
-            {
-                return null;
-            }
-        }
-
-        public string LogLevel { get; set; } = string.Empty;
-        public DateTime TimestampUtc { get; set; }
-        public string Message { get; set; } = string.Empty;
-        public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
-
-        /// <summary>
-        /// After too much time spent cursing at regexes, here's what I got:
-        /// Parses input string into 'key=value' pair, considerate of quoted (") values.
-        /// </summary>
-        private static Dictionary<string, string> SplitAttrs(string input)
-        {
-            input += " ";
-            var result = new Dictionary<string, string>();
-
-            var key = string.Empty;
-            var value = string.Empty;
-            var mode = 1;
-            var inQuote = false;
-
-            foreach (var c in input)
-            {
-                if (mode == 1)
-                {
-                    if (c == '=') mode = 2;
-                    else if (c == ' ')
-                    {
-                        if (string.IsNullOrEmpty(key)) continue;
-                        else
-                        {
-                            result.Add(key, string.Empty);
-                            key = string.Empty;
-                            value = string.Empty;
-                        }
-                    }
-                    else key += c;
-                }
-                else if (mode == 2)
-                {
-                    if (c == ' ' && !inQuote)
-                    {
-                        result.Add(key, value);
-                        key = string.Empty;
-                        value = string.Empty;
-                        mode = 1;
-                    }
-                    else if (c == '\"')
-                    {
-                        inQuote = !inQuote;
-                    }
-                    else
-                    {
-                        value += c;
-                    }
-                }
-            }
-
-            return result;
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -44,9 +44,7 @@ namespace CodexPlugin
             transferSpeeds = new TransferSpeeds();
         }
 
-        public RunningPod Pod { get { return CodexAccess.Container; } }
-        
-        public RunningContainer Container { get { return Pod.Containers.Single(); } }
+        public RunningContainer Container { get { return CodexAccess.Container; } }
         public CodexAccess CodexAccess { get; }
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
@@ -58,7 +56,7 @@ namespace CodexPlugin
         {
             get
             {
-                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
             }
         }
 
@@ -73,7 +71,7 @@ namespace CodexPlugin
 
         public string GetName()
         {
-            return Container.Name;
+            return CodexAccess.Container.Name;
         }
 
         public DebugInfo GetDebugInfo()
@@ -144,13 +142,11 @@ namespace CodexPlugin
 
         public void Stop(bool waitTillStopped)
         {
-            CrashWatcher.Stop();
-            Group.Stop(this, waitTillStopped);
-            // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-            //     "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-            //     "available for codex-nodes in groups of 1.");
-            //
-            // Group.BringOffline(waitTillStopped);
+            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+                "available for codex-nodes in groups of 1.");
+
+            Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
@@ -175,7 +171,7 @@ namespace CodexPlugin
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Pod);
+            var podInfo = workflow.GetPodInfo(peer.Container);
 
             return peerInfo.Addrs.Select(a => a
                 .Replace("0.0.0.0", podInfo.Ip))

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -35,7 +35,7 @@ namespace CodexPlugin
 
         private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
+            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
             return ethAccount.EthAddress;
         }

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -15,11 +15,11 @@ namespace CodexPlugin
     {
         private readonly CodexStarter starter;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.starter = starter;
             Containers = containers;
-            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
             Version = new DebugInfoVersion();
         }
 
@@ -39,14 +39,7 @@ namespace CodexPlugin
             Containers = null!;
         }
 
-        public void Stop(CodexNode node, bool waitTillStopped)
-        {
-            starter.Stop(node.Pod, waitTillStopped);
-            Nodes = Nodes.Where(n => n != node).ToArray();
-            Containers = Containers.Where(c => c != node.Pod).ToArray();
-        }
-
-        public RunningPod[] Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
         public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
@@ -81,9 +74,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
+        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
+            var watcher = factory.CreateCrashWatcher(c);
             var access = new CodexAccess(tools, c, watcher);
             return factory.CreateOnlineCodexNode(access, this);
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -32,13 +32,13 @@ namespace CodexPlugin
         {
         }
 
-        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
         {
             containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
             return codexStarter.WrapCodexContainers(coreInterface, containers);

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -52,7 +52,6 @@ namespace CodexPlugin
         public CodexLogLevel Libp2p { get; set; }
         public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
-        public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn;
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -19,7 +19,7 @@ namespace CodexPlugin
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public RunningPod[] BringOnline(CodexSetup codexSetup)
+        public RunningContainers[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -34,14 +34,14 @@ namespace CodexPlugin
             {
                 var podInfo = GetPodInfo(rc);
                 var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();
 
             return containers;
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
         {
             var codexNodeFactory = new CodexNodeFactory(pluginTools);
 
@@ -65,14 +65,6 @@ namespace CodexPlugin
             Log("Stopped.");
         }
 
-        public void Stop(RunningPod pod, bool waitTillStopped)
-        {
-            Log($"Stopping node...");
-            var workflow = pluginTools.CreateWorkflow();
-            workflow.Stop(pod, waitTillStopped);
-            Log("Stopped.");
-        }
-
         public string GetCodexId()
         {
             if (versionResponse != null) return versionResponse.Version;
@@ -93,27 +85,24 @@ namespace CodexPlugin
             return startupConfig;
         }
 
-        private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var futureContainers = new List<FutureContainers>();
+            var result = new List<RunningContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
+                result.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-
-            return futureContainers
-                .Select(f => f.WaitForOnline())
-                .ToArray();
+            return result.ToArray();
         }
 
-        private PodInfo GetPodInfo(RunningPod rc)
+        private PodInfo GetPodInfo(RunningContainers rc)
         {
             var workflow = pluginTools.CreateWorkflow();
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
 
@@ -130,10 +119,10 @@ namespace CodexPlugin
             return group;
         }
 
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
+        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
         {
             Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
+            foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
         }
 
         private void LogSeparator()

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -73,18 +73,11 @@ namespace CodexPlugin
                     "contracts",
                     "clock"
                 };
-                var jsonSerializeTopics = new[]
-                {
-                    "serde",
-                    "json",
-                    "serialization"
-                };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
-                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)};" +
-                    $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)}";
+                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);
         }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -56,7 +56,7 @@ namespace CodexPlugin
                 ProofProbability = ToDecInt(purchase.ProofProbability),
                 Reward = ToDecInt(purchase.PricePerSlotPerSecond),
                 Collateral = ToDecInt(purchase.RequiredCollateral),
-                Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
+                Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds),
                 Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
                 Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -83,7 +83,7 @@ namespace CodexPlugin
     {
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10);
         private DateTime? contractStartUtc;
 
         public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -213,7 +213,8 @@ components:
           description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
         expiry:
           type: string
-          description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
+          description: Number as decimal string that represents expiry time of the request (in unix timestamp)
+
     StorageAsk:
       type: object
       required:

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -7,9 +7,9 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
+        public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
-            Pod = pod;
+            Containers = containers;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
@@ -17,9 +17,9 @@ namespace GethPlugin
             PubKey = pubKey;
         }
 
-        public RunningPod Pod { get; }
+        public RunningContainers Containers { get; }
         [JsonIgnore]
-        public RunningContainer Container {  get { return Pod.Containers.Single(); } }
+        public RunningContainer Container {  get { return Containers.Containers.Single(); } }
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -5,7 +5,6 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -28,7 +27,6 @@ namespace GethPlugin
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
         BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
-        BlockTimeEntry GetBlockForNumber(ulong number);
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -162,11 +160,6 @@ namespace GethPlugin
             return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
         }
 
-        public BlockTimeEntry GetBlockForNumber(ulong number)
-        {
-            return StartInteraction().GetBlockForNumber(number);
-        }
-
         protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,24 +6,24 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
         {
-            return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
+            return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,15 +31,15 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
             return starter.CollectMetricsFor(scrapeTargets);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
         {
-            runningPod = SerializeGate.Gate(runningPod);
-            return starter.CreateAccessForTarget(runningPod, target);
+            runningContainer = SerializeGate.Gate(runningContainer);
+            return starter.CreateAccessForTarget(runningContainer, target);
         }
 
         public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,7 +16,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -25,16 +25,16 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
+            var runningContainers = workflow.Start(1, recipe, startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
         {
-            var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
+            var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
         }
 

ProjectPlugins/WakuPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,34 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static RunningContainers[] DeployWakuNodes(this CoreInterface ci, int number, Action<IWakuSetup> setup)
+        {
+            return Plugin(ci).DeployWakuNodes(number, setup);
+        }
+
+        public static IWakuNode WrapWakuContainer(this CoreInterface ci, RunningContainer container)
+        {
+            return Plugin(ci).WrapWakuContainer(container);
+        }
+
+        public static IWakuNode StartWakuNode(this CoreInterface ci)
+        {
+            return ci.StartWakuNode(s => { });
+        }
+
+        public static IWakuNode StartWakuNode(this CoreInterface ci, Action<IWakuSetup> setup)
+        {
+            var rc = ci.DeployWakuNodes(1, setup);
+            return ci.WrapWakuContainer(rc.First().Containers.First());
+        }
+
+        private static WakuPlugin Plugin(CoreInterface ci)
+        {
+            return ci.GetPlugin<WakuPlugin>();
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuApiTypes.cs

@@ -0,0 +1,8 @@
+namespace WakuPlugin
+{
+    public class DebugInfoResponse
+    {
+        public string[] listenAddresses { get; set; } = Array.Empty<string>();
+        public string enrUri { get; set; } = string.Empty;
+    }
+}

ProjectPlugins/WakuPlugin/WakuContainerRecipe.cs

@@ -0,0 +1,38 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using Utils;
+
+namespace WakuPlugin
+{
+    public class WakuContainerRecipe : ContainerRecipeFactory
+    { 
+        public override string AppName => "waku";
+        //public override string Image => "statusteam/nim-waku:deploy-wakuv2-test";
+        public override string Image => "thatbenbierens/nim-waku:try";
+        public static string RestPortTag = "REST_PORT";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<WakuSetup>();
+
+            SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
+
+            AddEnvVar("WAKUNODE2_LOG_LEVEL", "TRACE");
+            AddEnvVar("WAKUNODE2_REST", "1");
+            AddExposedPortAndVar("WAKUNODE2_REST_PORT", RestPortTag);
+            AddEnvVar("WAKUNODE2_REST_ADDRESS", "0.0.0.0");
+
+            AddInternalPortAndVar("WAKUNODE2_TCP_PORT");
+            AddEnvVar("WAKUNODE2_RPC_ADDRESS", "0.0.0.0");
+
+            AddEnvVar("WAKUNODE2_DISCV5_DISCOVERY", "1");
+            AddInternalPortAndVar("WAKUNODE2_DISCV5_UDP_PORT");
+            AddEnvVar("WAKUNODE2_DISCV5_ENR_AUTO_UPDATEY", "1");
+
+            if (!string.IsNullOrEmpty(config.BootstrapEnr))
+            {
+                AddEnvVar("WAKUNODE2_DISCV5_BOOTSTRAP_NODE", config.BootstrapEnr);
+            }
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuNode.cs

@@ -0,0 +1,53 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public interface IWakuNode : IHasContainer
+    {
+        DebugInfoResponse DebugInfo();
+        void SubscribeToTopic(string topic);
+        void SendMessage(string topic, string message);
+        string[] GetMessages(string topic);
+    }
+
+    public class WakuNode : IWakuNode
+    {
+        private readonly IPluginTools tools;
+
+        public WakuNode(IPluginTools tools, RunningContainer container)
+        {
+            this.tools = tools;
+            Container = container;
+        }
+
+        public RunningContainer Container { get; }
+
+        public DebugInfoResponse DebugInfo()
+        {
+            return Api().HttpGetJson<DebugInfoResponse>("debug/v1/info");
+        }
+
+        public void SubscribeToTopic(string topic)
+        {
+            var response = Api().HttpPostString<string>(route: "relay/v1/subscriptions", body: topic);
+        }
+
+        public void SendMessage(string topic, string message)
+        {
+            var response = Api().HttpPostString<string>($"relay/v1/messages/{topic}", message);
+        }
+
+        public string[] GetMessages(string topic)
+        {
+            var response = Api().HttpGetString($"relay/v1/messages/{topic}");
+            return new[] { "" };
+        }
+
+        private IEndpoint Api()
+        {
+            var address = Container.GetAddress(tools.GetLog(), WakuContainerRecipe.RestPortTag);
+            return tools.CreateHttp().CreateEndpoint(address, "", logAlias: "waku");
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuPlugin.cs

@@ -0,0 +1,71 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public class WakuPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
+    {
+        private readonly IPluginTools tools;
+        private readonly WakuStarter starter;
+
+        public WakuPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+            starter = new WakuStarter(tools);
+        }
+
+        public string LogPrefix => "(Waku) ";
+
+        public void Announce()
+        {
+            tools.GetLog().Log($"Loaded with Waku plugin.");
+        }
+
+        public void AddMetadata(IAddMetadata metadata)
+        {
+            //metadata.Add("codexid", codexStarter.GetCodexId());
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public RunningContainers[] DeployWakuNodes(int numberOfNodes, Action<IWakuSetup> setup)
+        {
+            return starter.Start(numberOfNodes, setup);
+        }
+
+        public IWakuNode WrapWakuContainer(RunningContainer container)
+        {
+            container = SerializeGate.Gate(container);
+            return starter.Wrap(container);
+        }
+
+        //public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        //{
+        //    containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
+        //    return codexStarter.WrapCodexContainers(coreInterface, containers);
+        //}
+
+        //public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
+        //{
+        //    var codexSetup = GetSetup(1, setup);
+        //    if (codexSetup.MarketplaceConfig == null) return;
+
+        //    var mconfig = codexSetup.MarketplaceConfig;
+        //    foreach (var node in result)
+        //    {
+        //        mconfig.GethNode.SendEth(node, mconfig.InitialEth);
+        //        mconfig.CodexContracts.MintTestTokens(mconfig.GethNode, node, mconfig.InitialTokens);
+        //    }
+        //}
+
+        //private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
+        //{
+        //    var codexSetup = new CodexSetup(numberOfNodes);
+        //    codexSetup.LogLevel = defaultLogLevel;
+        //    setup(codexSetup);
+        //    return codexSetup;
+        //}
+    }
+}

ProjectPlugins/WakuPlugin/WakuPlugin.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/WakuPlugin/WakuSetup.cs

@@ -0,0 +1,26 @@
+namespace WakuPlugin
+{
+    public interface IWakuSetup
+    {
+        IWakuSetup WithName(string name);
+        IWakuSetup WithBootstrapNode(IWakuNode node);
+    }
+
+    public class WakuSetup : IWakuSetup
+    {
+        internal string? Name { get; private set; }
+        internal string? BootstrapEnr { get; private set; }
+
+        public IWakuSetup WithName(string name)
+        {
+            Name = name;
+            return this;
+        }
+
+        public IWakuSetup WithBootstrapNode(IWakuNode node)
+        {
+            BootstrapEnr = node.DebugInfo().enrUri;
+            return this;
+        }
+    }
+}

ProjectPlugins/WakuPlugin/WakuStarter.cs

@@ -0,0 +1,45 @@
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+
+namespace WakuPlugin
+{
+    public class WakuStarter
+    {
+        private readonly IPluginTools tools;
+
+        public WakuStarter(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public RunningContainers[] Start(int numberOfNodes, Action<IWakuSetup> setup)
+        {
+            var result = new List<RunningContainers>();
+            var workflow = tools.CreateWorkflow();
+            var startupConfig = CreateStartupConfig(setup);
+
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(workflow.Start(1, new WakuContainerRecipe(), startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        public IWakuNode Wrap(RunningContainer container)
+        {
+            return new WakuNode(tools, container);
+        }
+
+        private StartupConfig CreateStartupConfig(Action<IWakuSetup> setup)
+        {
+            var config = new WakuSetup();
+            setup(config);
+            var startupConfig = new StartupConfig();
+            startupConfig.Add(config);
+            startupConfig.NameOverride = config.Name;
+            return startupConfig;
+        }
+    }
+}

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -49,8 +49,8 @@ namespace ContinuousTests
             var start = startUtc.ToString("o");
             var end = endUtc.ToString("o");
 
-            var containerName = container.RunningPod.StartResult.Deployment.Name;
-            var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
+            var containerName = container.RunningContainers.StartResult.Deployment.Name;
+            var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
 
             //container_name : codex3-5 - deploymentName as stored in pod
             // pod_namespace : codex - continuous - nolimits - tests - 1

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -125,8 +125,8 @@ namespace ContinuousTests
             foreach (var node in nodes)
             {
                 var container = node.Container;
-                var deploymentName = container.RunningPod.StartResult.Deployment.Name;
-                var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
+                var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
+                var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
                 elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
@@ -295,13 +295,13 @@ namespace ContinuousTests
             return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
         }
 
-        private RunningPod[] SelectRandomContainers()
+        private RunningContainers[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
+            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
             if (number == -1) return containers.ToArray();
 
-            var result = new RunningPod[number];
+            var result = new RunningContainers[number];
             for (var i = 0; i < number; i++)
             {
                 result[i] = containers.PickOneRandom();

Tests/CodexContinuousTests/StartupChecker.cs

@@ -43,13 +43,13 @@ namespace ContinuousTests
             var workflow = entryPoint.Tools.CreateWorkflow();
             foreach (var instance in deployment.CodexInstances)
             {
-                foreach (var container in instance.Pod.Containers)
+                foreach (var container in instance.Containers.Containers)
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
                     log.Log(
-                        $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
+                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
@@ -92,7 +92,7 @@ namespace ContinuousTests
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
             var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,4 +1,3 @@
-using CodexPlugin;
 using CodexTests;
 using DistTestCore;
 using FileUtils;
@@ -8,45 +7,35 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class DownloadTests : AutoBootstrapDistTest
+    public class DownloadTests : CodexDistTest
     {
-        [Test]
-        [Combinatorial]
+        [TestCase(3, 500)]
+        [TestCase(5, 100)]
+        [TestCase(10, 256)]
         [UseLongTimeouts]
-        public void ParallelDownload(
-            [Values(1, 3, 5)] int numberOfFiles,
-            [Values(10, 50, 100)] int filesizeMb)
+        public void ParallelDownload(int numberOfNodes, int filesizeMb)
         {
-            var host = StartCodex();
-            var client = StartCodex();
+            var group = AddCodex(numberOfNodes);
+            var host = AddCodex();
 
-            var testfiles = new List<TrackedFile>();
-            var contentIds = new List<ContentId>();
-            var downloadedFiles = new List<TrackedFile?>();
-
-            for (int i = 0; i < numberOfFiles; i++)
+            foreach (var node in group)
             {
-                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
-                contentIds.Add(new ContentId());
-                downloadedFiles.Add(null);
+                host.ConnectToPeer(node);
             }
 
-            for (int i = 0; i < numberOfFiles; i++)
+            var testFile = GenerateTestFile(filesizeMb.MB());
+            var contentId = host.UploadFile(testFile);
+            var list = new List<Task<TrackedFile?>>();
+
+            foreach (var node in group)
             {
-                contentIds[i] = host.UploadFile(testfiles[i]);
+                list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));
             }
 
-            var downloadTasks = new List<Task>();
-            for (int i = 0; i < numberOfFiles; i++)
+            Task.WaitAll(list.ToArray());
+            foreach (var task in list)
             {
-                downloadTasks.Add(Task.Run(() => { downloadedFiles[i] = client.DownloadContent(contentIds[i]); }));
-            }
-
-            Task.WaitAll(downloadTasks.ToArray());
-
-            for (int i = 0; i < numberOfFiles; i++)
-            {
-                testfiles[i].AssertIsEqual(downloadedFiles[i]);
+                testFile.AssertIsEqual(task.Result);
             }
         }
     }

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -48,9 +48,7 @@ namespace CodexLongTests.BasicTests
 
             var expectedFile = GenerateTestFile(sizeMB);
 
-            var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
-
-            Time.SetRetryFailedCallback(i => OnFailed(i, node));
+            var node = AddCodex(s => s.WithStorageQuota((size + 10).MB()));
 
             var uploadStart = DateTime.UtcNow;
             var cid = node.UploadFile(expectedFile);
@@ -62,17 +60,6 @@ namespace CodexLongTests.BasicTests
             AssertTimeConstraint(uploadStart, downloadStart, downloadFinished, size);
         }
 
-        private void OnFailed(int tries, ICodexNode node)
-        {
-            if (tries < 5) return;
-
-            if (tries % 10 == 0)
-            {
-                Log($"After try {tries}, downloading node log.");
-                Ci.DownloadLog(node);
-            }
-        }
-
         private void AssertTimeConstraint(DateTime uploadStart, DateTime downloadStart, DateTime downloadFinished, long size)
         {
             float sizeInMB = size;

Tests/CodexLongTests/BasicTests/TestInfraTests.cs

@@ -6,12 +6,10 @@ namespace CodexLongTests.BasicTests
 {
     public class TestInfraTests : CodexDistTest
     {
-        [Test]
-        [UseLongTimeouts]
-        [Ignore("Not supported atm")]
+        [Test, UseLongTimeouts]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = StartCodex(1000, s => s.EnableMetrics());
+            var group = AddCodex(1000, s => s.EnableMetrics());
 
             var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray();
 
@@ -19,14 +17,12 @@ namespace CodexLongTests.BasicTests
                 "Not all created nodes provided a unique id.");
         }
 
-        [Test]
-        [UseLongTimeouts]
-        [Ignore("Not supported atm")]
+        [Test, UseLongTimeouts]
         public void TestInfraSupportsManyConcurrentPods()
         {
             for (var i = 0; i < 20; i++)
             {
-                var n = StartCodex();
+                var n = AddCodex();
 
                 Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id));
             }

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -8,39 +8,41 @@ using Utils;
 namespace CodexLongTests.BasicTests
 {
     [TestFixture]
-    public class UploadTests : AutoBootstrapDistTest
+    public class UploadTests : CodexDistTest
     {
-        [Test]
-        [Combinatorial] 
+        [TestCase(3, 50)]
+        [TestCase(5, 75)]
+        [TestCase(10, 25)]
         [UseLongTimeouts]
-        public void ParallelUpload(
-            [Values(1, 3, 5)] int numberOfFiles,
-            [Values(10, 50, 100)] int filesizeMb)
+        public void ParallelUpload(int numberOfNodes, int filesizeMb)
         {
-            var host = StartCodex();
-            var client = StartCodex();
+            var group = AddCodex(numberOfNodes);
+            var host = AddCodex();
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
 
             var testfiles = new List<TrackedFile>();
-            var contentIds = new List<ContentId>();
+            var contentIds = new List<Task<ContentId>>();
 
-            for (int i = 0; i < numberOfFiles; i++)
+            for (int i = 0; i < group.Count(); i++)
             {
                 testfiles.Add(GenerateTestFile(filesizeMb.MB()));
-                contentIds.Add(new ContentId());
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
             }
-
-            var uploadTasks = new List<Task>();
-            for (int i = 0; i < numberOfFiles; i++)
+            var downloads = new List<Task<TrackedFile?>>();
+            for (int i = 0; i < group.Count(); i++)
             {
-                uploadTasks.Add(Task.Run(() => { contentIds[i] = host.UploadFile(testfiles[i]); }));
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); }));
             }
-
-            Task.WaitAll(uploadTasks.ToArray());
-
-            for (int i = 0; i < numberOfFiles; i++)
+            Task.WaitAll(downloads.ToArray());
+            for (int i = 0; i < group.Count(); i++)
             {
-                var downloaded = client.DownloadContent(contentIds[i]);
-                testfiles[i].AssertIsEqual(downloaded);
+                testfiles[i].AssertIsEqual(downloads[i].Result);
             }
         }
     }

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests
             [Values(10, 15, 20)] int numberOfNodes,
             [Values(10, 100)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) StartCodex();
+            for (var i = 0; i < numberOfNodes; i++) AddCodex();
 
             CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -1,120 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.ScalabilityTests
-{
-    [TestFixture]
-    public class MultiPeerDownloadTests : AutoBootstrapDistTest
-    {
-        [Test]
-        [DontDownloadLogs]
-        [UseLongTimeouts]
-        [Combinatorial]
-        public void MultiPeerDownload(
-            [Values(5, 10, 20)] int numberOfHosts,
-            [Values(100, 1000)] int fileSize
-        )
-        {
-            var hosts = StartCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
-            var file = GenerateTestFile(fileSize.MB());
-            var cid = hosts[0].UploadFile(file);
-            var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
-
-            var uploadLog = Ci.DownloadLog(hosts[0]);
-            var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
-            var blockCids = uploadLog
-                .FindLinesThatContain("Putting block into network store")
-                .Select(s =>
-                {
-                    var start = s.IndexOf("cid=") + 4;
-                    var end = s.IndexOf(" count=");
-                    var len = end - start;
-                    return s.Substring(start, len);
-                })
-                .ToArray();
-
-            Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
-
-            foreach (var h in hosts) h.DownloadContent(cid);
-
-            var client = StartCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
-            var resultFile = client.DownloadContent(cid);
-            resultFile!.AssertIsEqual(file);
-
-            var downloadLog = Ci.DownloadLog(client);
-            var host = string.Empty;
-            var blockCidHostMap = new Dictionary<string, string>();
-            downloadLog.IterateLines(line =>
-            {
-                if (line.Contains("peer=") && line.Contains(" len="))
-                {
-                    var start = line.IndexOf("peer=") + 5;
-                    var end = line.IndexOf(" len=");
-                    var len = end - start;
-                    host = line.Substring(start, len);
-                }
-                else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
-                {
-                    var start = line.IndexOf("cid=") + 4;
-                    var end = line.IndexOf(" count=");
-                    var len = end - start;
-                    var blockCid = line.Substring(start, len);
-
-                    blockCidHostMap.Add(blockCid, host);
-                    host = string.Empty;
-                }
-            });
-
-            var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
-            //PrintFullMap(blockCidHostMap);
-            PrintOverview(blockCidHostMap);
-
-            Log("Expected number of blocks: " + expectedNumberOfBlocks);
-            Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
-            Log("Total blocks fetched by hosts: " + totalFetched);
-            Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
-        }
-
-        private void PrintOverview(Dictionary<string, string> blockCidHostMap)
-        {
-            var overview = new Dictionary<string, int>();
-            foreach (var pair in blockCidHostMap)
-            {
-                if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
-                else overview[pair.Value]++;
-            }
-
-            Log("Blocks fetched per host:");
-            foreach (var pair in overview)
-            {
-                Log($"Host: {pair.Key} = {pair.Value}");
-            }
-        }
-
-        private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
-        {
-            Log("Per block, host it was fetched from:");
-            foreach (var pair in blockCidHostMap)
-            {
-                if (string.IsNullOrEmpty(pair.Value))
-                {
-                    Log($"block: {pair.Key} = Not seen");
-                }
-                else
-                {
-                    Log($"block: {pair.Key} = '{pair.Value}'");
-                }
-            }
-        }
-
-        private long RoundUp(long filesize, long blockSize)
-        {
-            double f = filesize;
-            double b = blockSize;
-
-            var result = Math.Ceiling(f / b);
-            return Convert.ToInt64(result);
-        }
-    }
-}

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -1,120 +0,0 @@
-using CodexPlugin;
-using DistTestCore;
-using FileUtils;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.ScalabilityTests;
-
-[TestFixture]
-public class ScalabilityTests : CodexDistTest
-{
-    /// <summary>
-    /// We upload a file to node A, then download it with B.
-    /// Then we stop node A, and download again with node C.
-    /// </summary>
-    [Test]
-    [Combinatorial]
-    [UseLongTimeouts]
-    [DontDownloadLogs]
-    public void ShouldMaintainFileInNetwork(
-        [Values(10, 40, 80, 100)] int numberOfNodes,
-        [Values(100, 1000, 5000, 10000)] int fileSizeInMb
-    )
-    {
-        var logLevel = CodexLogLevel.Info;
-
-        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
-        var nodes = StartCodex(numberOfNodes - 1, s => s
-            .WithBootstrapNode(bootstrap)
-            .WithLogLevel(logLevel)
-            .WithStorageQuota((fileSizeInMb + 50).MB())
-        ).ToList();
-
-        var uploader = nodes.PickOneRandom();
-        var downloader = nodes.PickOneRandom();
-
-        var testFile = GenerateTestFile(fileSizeInMb.MB());
-        var contentId = uploader.UploadFile(testFile);
-        var downloadedFile = downloader.DownloadContent(contentId);
-
-        downloadedFile!.AssertIsEqual(testFile);
-
-        uploader.Stop(true);
-
-        var otherDownloader = nodes.PickOneRandom();
-        downloadedFile = otherDownloader.DownloadContent(contentId);
-
-        downloadedFile!.AssertIsEqual(testFile);
-    }
-
-    /// <summary>
-    /// We upload a file to each node, to put a more wide-spread load on the network.
-    /// Then we run the same test as ShouldMaintainFileInNetwork.
-    /// </summary>
-    [Ignore("Fix ShouldMaintainFileInNetwork for all values first")]
-    [Test]
-    [Combinatorial]
-    [UseLongTimeouts]
-    [DontDownloadLogs]
-    public void EveryoneGetsAFile(
-        [Values(10, 40, 80, 100)] int numberOfNodes,
-        [Values(100, 1000, 5000, 10000)] int fileSizeInMb
-    )
-    {
-        var logLevel = CodexLogLevel.Info;
-
-        var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
-        var nodes = StartCodex(numberOfNodes - 1, s => s
-            .WithBootstrapNode(bootstrap)
-            .WithLogLevel(logLevel)
-            .WithStorageQuota((fileSizeInMb + 50).MB())
-        ).ToList();
-
-        var pairTasks = nodes.Select(n =>
-        {
-            return Task.Run(() =>
-            {
-                var file = GenerateTestFile(fileSizeInMb.MB());
-                var cid = n.UploadFile(file);
-                return new NodeFilePair(n, file, cid);
-            });
-        });
-
-        var pairs = pairTasks.Select(t => Time.Wait(t)).ToList();
-
-        RunDoubleDownloadTest(
-            pairs.PickOneRandom(),
-            pairs.PickOneRandom(),
-            pairs.PickOneRandom()
-        );
-    }
-
-    private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2)
-    {
-        var expectedFile = source.File;
-        var cid = source.Cid;
-
-        var file1 = dl1.Node.DownloadContent(cid);
-        file1!.AssertIsEqual(expectedFile);
-
-        source.Node.Stop(true);
-
-        var file2 = dl2.Node.DownloadContent(cid);
-        file2!.AssertIsEqual(expectedFile);
-    }
-
-    public class NodeFilePair
-    {
-        public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid)
-        {
-            Node = node;
-            File = file;
-            Cid = cid;
-        }
-
-        public ICodexNode Node { get; }
-        public TrackedFile File { get; }
-        public ContentId Cid { get; }
-    }
-}

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -8,7 +8,7 @@ namespace CodexTests
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
+            BootstrapNode = AddCodex(s => s.WithName("BOOTSTRAP"));
         }
 
         [TearDown]

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -0,0 +1,298 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using KubernetesWorkflow.Types;
+using Logging;
+using MetricsPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [Ignore("Used for debugging continuous tests")]
+    [TestFixture]
+    public class ContinuousSubstitute : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ContinuousTestSubstitute()
+        {
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth"));
+            var contract = Ci.StartCodexContracts(geth);
+
+            var group = AddCodex(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(geth, contract, s => s
+                        .WithInitial(10.Eth(), 100000.TestTokens())
+                        .AsStorageNode()
+                        .AsValidator())
+                    .WithBlockTTL(TimeSpan.FromMinutes(5))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10))
+                    .WithBlockMaintenanceNumber(100)
+                    .WithStorageQuota(1.GB()));
+
+            var nodes = group.Cast<CodexNode>().ToArray();
+
+            var rc = Ci.DeployMetricsCollector(nodes);
+
+            var availability = new StorageAvailability(
+                totalSpace: 500.MB(),
+                maxDuration: TimeSpan.FromMinutes(5),
+                minPriceForTotalSpace: 500.TestTokens(),
+                maxCollateral: 1024.TestTokens()
+            );
+
+            foreach (var node in nodes)
+            {
+                node.Marketplace.MakeStorageAvailable(availability);
+            }
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                var allNodes = nodes.ToList();
+                var primary = allNodes.PickOneRandom();
+                var secondary = allNodes.PickOneRandom();
+
+                Log("Run Test");
+                PerformTest(primary, secondary, rc);
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void LogBytesPerMillisecond(Action action)
+        {
+            var sw = Stopwatch.Begin(GetTestLog());
+            action();
+            var duration = sw.End();
+            double totalMs = duration.TotalMilliseconds;
+            double totalBytes = fileSize.SizeInBytes;
+
+            var bytesPerMs = totalBytes / totalMs;
+            Log($"Bytes per millisecond: {bytesPerMs}");
+        }
+
+        [Test]
+        public void PeerTest()
+        {
+            var group = AddCodex(5, o => o
+                    //.EnableMetrics()
+                    //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(1.GB()));
+
+            var nodes = group.Cast<CodexNode>().ToArray();
+
+            var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+                //CheckRoutingTables(GetAllOnlineCodexNodes());
+
+                var node = nodes.ToList().PickOneRandom();
+                var file = GenerateTestFile(50.MB());
+                node.UploadFile(file);
+
+                Thread.Sleep(20000);
+            }
+        }
+
+        private void CheckRoutingTables(IEnumerable<ICodexNode> nodes)
+        {
+            var all = nodes.ToArray();
+            var allIds = all.Select(n => n.GetDebugInfo().Table.LocalNode.NodeId).ToArray();
+
+            var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail(string.Join(Environment.NewLine, errors));
+            }
+        }
+
+        private string AreAllPresent(ICodexNode n, string[] allIds)
+        {
+            var info = n.GetDebugInfo();
+            var known = info.Table.Nodes.Select(n => n.NodeId).ToArray();
+            var expected = allIds.Where(i => i != info.Table.LocalNode.NodeId).ToArray();
+
+            if (!expected.All(ex => known.Contains(ex)))
+            {
+                return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
+            }
+
+            return string.Empty;
+        }
+
+        private ByteSize fileSize = 80.MB();
+
+        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
+
+        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc)
+        {
+            ScopedTestFiles(() =>
+            {
+                var testFile = GenerateTestFile(fileSize);
+
+                var metrics = Ci.WrapMetricsCollector(rc, primary);
+                var beforeBytesStored = metrics.GetMetric(BytesStoredMetric);
+
+                ContentId contentId = null!;
+                LogBytesPerMillisecond(() => contentId = primary.UploadFile(testFile));
+
+                var low = fileSize.SizeInBytes;
+                var high = low * 1.2;
+                Log("looking for: " + low + " < " + high);
+
+                Time.WaitUntil(() =>
+                {
+                    var afterBytesStored = metrics.GetMetric(BytesStoredMetric);
+                    var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
+
+                    return high > newBytes && newBytes > low;
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
+
+                FileUtils.TrackedFile? downloadedFile = null;
+                LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId));
+
+                testFile.AssertIsEqual(downloadedFile);
+            });
+        }
+
+        [Test]
+        public void HoldMyBeerTest()
+        {
+            var blockExpirationTime = TimeSpan.FromMinutes(3);
+            var group = AddCodex(3, o => o
+                    .EnableMetrics()
+                    .WithBlockTTL(blockExpirationTime)
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithStorageQuota(2000.MB()));
+
+            var nodes = group.Cast<CodexNode>().ToArray();
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
+
+            var filesize = 80.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            var sizeInBytes = filesize.SizeInBytes;
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            var startTime = DateTime.UtcNow;
+            var successfulUploads = 0;
+            var successfulDownloads = 0;
+
+            while (DateTime.UtcNow < endTime)
+            {
+                foreach (var node in nodes)
+                {
+                    try
+                    {
+                        Thread.Sleep(TimeSpan.FromSeconds(5));
+
+                        ScopedTestFiles(() =>
+                        {
+                            var uploadStartTime = DateTime.UtcNow;
+                            var file = GenerateTestFile(filesize);
+                            var cid = node.UploadFile(file);
+
+                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
+                            Measure("upload-log-asserts", () =>
+                            {
+                                var uploadLog = Ci.DownloadLog(node, tailLines: 50000);
+
+                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                                uploadLog.DeleteFile();
+
+                                var storeLine = GetLineForCidTag(storeLines, cidTag);
+                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                            });
+                            successfulUploads++;
+
+                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
+                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
+                            {
+                                Assert.Fail("Upload took too long. Blocks already expired.");
+                            }
+
+                            var dl = node.DownloadContent(cid);
+                            file.AssertIsEqual(dl);
+
+                            Measure("download-log-asserts", () =>
+                            {
+                                var downloadLog = Ci.DownloadLog(node, tailLines: 50000);
+
+                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                                downloadLog.DeleteFile();
+
+                                var sentLine = GetLineForCidTag(sentLines, cidTag);
+                                AssertSentLineContains(sentLine, sizeInBytes);
+                            });
+                            successfulDownloads++;
+                        });
+                    }
+                    catch
+                    {
+                        var testDuration = DateTime.UtcNow - startTime;
+                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
+                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
+                        Thread.Sleep(TimeSpan.FromSeconds(30));
+                        throw;
+                    }
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void AssertSentLineContains(string sentLine, long sizeInBytes)
+        {
+            var tag = "bytes=";
+            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
+            var bytes = Convert.ToInt64(token);
+            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
+        }
+
+        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
+        {
+            var tokens = storeLine.Split(" ");
+
+            var blocksToken = GetToken(tokens, "blocks=");
+            var sizeToken = GetToken(tokens, "size=");
+            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
+            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
+
+            var blocks = Convert.ToInt64(blocksToken);
+            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
+
+            var lineLog = $" Line: '{storeLine}'";
+            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
+            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
+        }
+
+        private string GetLineForCidTag(string[] lines, string cidTag)
+        {
+            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
+            if (result == null)
+            {
+                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
+                throw new Exception();
+            }
+
+            return result;
+        }
+
+        private string? GetToken(string[] tokens, string tag)
+        {
+            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
+            if (token == null) return null;
+            return token.Substring(tag.Length);
+        }
+    }
+}

Tests/CodexTests/BasicTests/DiscordBotTests.cs

@@ -5,13 +5,12 @@ using GethPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.UtilityTests
+namespace CodexTests.BasicTests
 {
     [TestFixture]
     public class DiscordBotTests : AutoBootstrapDistTest
     {
         [Test]
-        [Ignore("Used for debugging bots")]
         public void BotRewardTest()
         {
             var myAccount = EthAccount.GenerateNew();
@@ -58,7 +57,7 @@ namespace CodexTests.UtilityTests
 
             for (var i = 0; i < numberOfHosts; i++)
             {
-                var seller = StartCodex(s => s
+                var seller = AddCodex(s => s
                     .WithName("Seller")
                     .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
                     {
@@ -82,7 +81,7 @@ namespace CodexTests.UtilityTests
 
             var testFile = GenerateTestFile(fileSize);
 
-            var buyer = StartCodex(s => s
+            var buyer = AddCodex(s => s
                 .WithName("Buyer")
                 .EnableMarketplace(geth, contracts, m => m
                     .WithAccount(myAccount)

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -13,7 +13,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
+            var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
 
             var cid = primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -28,8 +28,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoMetricsExample()
         {
-            var group = StartCodex(2, s => s.EnableMetrics());
-            var group2 = StartCodex(2, s => s.EnableMetrics());
+            var group = AddCodex(2, s => s.EnableMetrics());
+            var group2 = AddCodex(2, s => s.EnableMetrics());
 
             var primary = group[0];
             var secondary = group[1];

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -19,24 +19,23 @@ namespace CodexTests.BasicTests
 
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
-            
-            var numberOfHosts = 5;
-            var hosts = StartCodex(numberOfHosts, s => s
-                .WithName("Host")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                {
-                    ContractClock = CodexLogLevel.Trace,
-                })
-                .WithStorageQuota(11.GB())
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), hostInitialBalance)
-                    .AsStorageNode()
-                    .AsValidator()));
 
-            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
-            foreach (var host in hosts)
+            var numberOfHosts = 3;
+            for (var i = 0; i < numberOfHosts; i++)
             {
-                AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
+                var host = AddCodex(s => s
+                    .WithName("Host")
+                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                    {
+                        ContractClock = CodexLogLevel.Trace,
+                    })
+                    .WithStorageQuota(11.GB())
+                    .EnableMarketplace(geth, contracts, m => m
+                        .WithInitial(10.Eth(), hostInitialBalance)
+                        .AsStorageNode()
+                        .AsValidator()));
+
+                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
 
                 var availability = new StorageAvailability(
                     totalSpace: 10.GB(),
@@ -49,7 +48,7 @@ namespace CodexTests.BasicTests
 
             var testFile = GenerateTestFile(fileSize);
 
-            var client = StartCodex(s => s
+            var client = AddCodex(s => s
                 .WithName("Client")
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), clientInitialBalance)));
@@ -95,7 +94,7 @@ namespace CodexTests.BasicTests
                 Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
 
                 if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
-            }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
+            }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
         }
 
         private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)

Tests/CodexTests/BasicTests/NetworkIsolationTest.cs

@@ -3,7 +3,7 @@ using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.UtilityTests
+namespace CodexTests.BasicTests
 {
     // Warning!
     // This is a test to check network-isolation in the test-infrastructure.
@@ -19,7 +19,7 @@ namespace CodexTests.UtilityTests
         {
             node = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait));
+            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
         }
 
         [Test]
@@ -27,7 +27,7 @@ namespace CodexTests.UtilityTests
         {
             var myNode = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects));
+            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
 
             try
             {

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,4 +1,5 @@
 using CodexPlugin;
+using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
@@ -10,7 +11,7 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = StartCodex();
+            var primary = Ci.StartCodexNode();
 
             PerformOneClientTest(primary);
         }
@@ -18,11 +19,11 @@ namespace CodexTests.BasicTests
         [Test]
         public void RestartTest()
         {
-            var primary = StartCodex();
+            var primary = Ci.StartCodexNode();
 
             primary.Stop(waitTillStopped: true);
 
-            primary = StartCodex();
+            primary = Ci.StartCodexNode();
 
             PerformOneClientTest(primary);
         }

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -9,8 +9,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void ThreeClient()
         {
-            var primary = StartCodex();
-            var secondary = StartCodex();
+            var primary = AddCodex();
+            var secondary = AddCodex();
 
             var testFile = GenerateTestFile(10.MB());
 

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -10,8 +10,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void TwoClientTest()
         {
-            var uploader = StartCodex(s => s.WithName("Uploader"));
-            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
+            var uploader = AddCodex(s => s.WithName("Uploader"));
+            var downloader = AddCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
 
             PerformTwoClientTest(uploader, downloader);
         }

Tests/CodexTests/CodexDistTest.cs

@@ -39,22 +39,22 @@ namespace CodexTests
             onlineCodexNodes.Remove(lifecycle);
         }
 
-        public ICodexNode StartCodex()
+        public ICodexNode AddCodex()
         {
-            return StartCodex(s => { });
+            return AddCodex(s => { });
         }
 
-        public ICodexNode StartCodex(Action<ICodexSetup> setup)
+        public ICodexNode AddCodex(Action<ICodexSetup> setup)
         {
-            return StartCodex(1, setup)[0];
+            return AddCodex(1, setup)[0];
         }
 
-        public ICodexNodeGroup StartCodex(int numberOfNodes)
+        public ICodexNodeGroup AddCodex(int numberOfNodes)
         {
-            return StartCodex(numberOfNodes, s => { });
+            return AddCodex(numberOfNodes, s => { });
         }
 
-        public ICodexNodeGroup StartCodex(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexNodeGroup AddCodex(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var group = Ci.StartCodexNodes(numberOfNodes, s =>
             {

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -11,7 +11,7 @@ namespace CodexTests.DownloadConnectivityTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDownload()
         {
-            StartCodex(2, s => s.EnableMetrics());
+            AddCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -21,7 +21,7 @@ namespace CodexTests.DownloadConnectivityTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TestTokens())));
 
             AssertAllNodesConnected();
@@ -33,7 +33,7 @@ namespace CodexTests.DownloadConnectivityTests
             [Values(2, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            StartCodex(numberOfNodes);
+            AddCodex(numberOfNodes);
 
             AssertAllNodesConnected(sizeMBs);
         }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -8,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void TwoLayersTest()
         {
-            var root = StartCodex();
-            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
-            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
-            var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
+            var root = AddCodex();
+            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
+            var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node));
 
             AssertAllNodesConnected();
         }
@@ -19,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void ThreeLayersTest()
         {
-            var root = StartCodex();
-            var l1Source = StartCodex(s => s.WithBootstrapNode(root));
-            var l1Node = StartCodex(s => s.WithBootstrapNode(root));
-            var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
-            var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
+            var root = AddCodex();
+            var l1Source = AddCodex(s => s.WithBootstrapNode(root));
+            var l1Node = AddCodex(s => s.WithBootstrapNode(root));
+            var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node));
+            var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node));
 
             AssertAllNodesConnected();
         }
@@ -33,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void NodeChainTest(int chainLength)
         {
-            var node = StartCodex();
+            var node = AddCodex();
             for (var i = 1; i < chainLength; i++)
             {
-                node = StartCodex(s => s.WithBootstrapNode(node));
+                node = AddCodex(s => s.WithBootstrapNode(node));
             }
 
             AssertAllNodesConnected();

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -12,7 +12,7 @@ namespace CodexTests.PeerDiscoveryTests
         public void CanReportUnknownPeerId()
         {
             var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
-            var node = StartCodex();
+            var node = AddCodex();
 
             var result = node.GetDebugPeer(unknownId);
             Assert.That(result.IsPeerFound, Is.False);
@@ -21,7 +21,7 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDiscovery()
         {
-            StartCodex(2, s => s.EnableMetrics());
+            AddCodex(2, s => s.EnableMetrics());
 
             AssertAllNodesConnected();
         }
@@ -31,7 +31,7 @@ namespace CodexTests.PeerDiscoveryTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+            AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TestTokens())));
 
             AssertAllNodesConnected();
@@ -42,7 +42,7 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void VariableNodes(int number)
         {
-            StartCodex(number);
+            AddCodex(number);
 
             AssertAllNodesConnected();
         }

Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs

@@ -1,84 +0,0 @@
-using DistTestCore;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.ScalabilityTests
-{
-    [TestFixture]
-    public class ClusterDiscSpeedTests : DistTest
-    {
-        private readonly Random random = new Random();
-
-        [Test]
-        [Combinatorial]
-        [Ignore("Used to measure disc io speeds in cluster.")]
-        public void DiscSpeedTest(
-            [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb
-        )
-        {
-            long targetSize = (long)(1024 * 1024 * 1024) * 2;
-            long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
-
-            var filename = nameof(DiscSpeedTest);
-
-            Thread.Sleep(2000);
-            if (File.Exists(filename)) File.Delete(filename);
-            Thread.Sleep(2000);
-            var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
-            Thread.Sleep(2000);
-            var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
-            
-            Log($"Write speed: {writeSpeed} per second.");
-            Log($"Read speed: {readSpeed} per second.");
-        }
-
-        private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename)
-        {
-            long bytesWritten = 0;
-            var buffer = new byte[bufferSizeBytes];
-            random.NextBytes(buffer);
-
-            var sw = Stopwatch.Begin(GetTestLog());
-            using (var stream = File.OpenWrite(filename))
-            {
-                while (bytesWritten < targetSize)
-                {
-                    long remaining = targetSize - bytesWritten;
-                    long toWrite = Math.Min(bufferSizeBytes, remaining);
-
-                    stream.Write(buffer, 0, Convert.ToInt32(toWrite));
-                    bytesWritten += toWrite;
-                }
-            }
-            var duration = sw.End("WriteTime");
-            double totalSeconds = duration.TotalSeconds;
-            double totalBytes = bytesWritten;
-            double bytesPerSecond = totalBytes / totalSeconds;
-            return new ByteSize(Convert.ToInt64(bytesPerSecond));
-        }
-
-        private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename)
-        {
-            long bytesRead = 0;
-            var buffer = new byte[bufferSizeBytes];
-            var sw = Stopwatch.Begin(GetTestLog());
-            using (var stream = File.OpenRead(filename))
-            {
-                while (bytesRead < targetSize)
-                {
-                    long remaining = targetSize - bytesRead;
-                    long toRead = Math.Min(bufferSizeBytes, remaining);
-
-                    var r = stream.Read(buffer, 0, Convert.ToInt32(toRead));
-                    bytesRead += r;
-                }
-            }
-            var duration = sw.End("ReadTime");
-            double totalSeconds = duration.TotalSeconds;
-            double totalBytes = bytesRead;
-            double bytesPerSecond = totalBytes / totalSeconds;
-            return new ByteSize(Convert.ToInt64(bytesPerSecond));
-        }
-    }
-}

Tests/CodexTests/UtilityTests/LogHelperTests.cs

@@ -1,56 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.UtilityTests
-{
-    [TestFixture]
-    public class LogHelperTests : AutoBootstrapDistTest
-    {
-        [Test]
-        [Ignore("Used to find the most common log messages.")]
-        public void FindMostCommonLogMessages()
-        {
-            var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace));
-            var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace));
-
-            var cid = uploader.UploadFile(GenerateTestFile(100.MB()));
-
-            Thread.Sleep(1000);
-            var logStartUtc = DateTime.UtcNow;
-            Thread.Sleep(1000);
-
-            downloader.DownloadContent(cid);
-
-            var map = GetLogMap(downloader, logStartUtc).OrderByDescending(p => p.Value);
-            Log("Downloader - Receive");
-            foreach (var entry in map)
-            {
-                if (entry.Value > 9)
-                {
-                    Log($"'{entry.Key}' = {entry.Value}");
-                }
-            }
-        }
-
-        private Dictionary<string, int> GetLogMap(ICodexNode node, DateTime? startUtc = null)
-        {
-            var log = Ci.DownloadLog(node);
-            var map = new Dictionary<string, int>();
-            log.IterateLines(line =>
-            {
-                var log = CodexLogLine.Parse(line);
-                if (log == null) return;
-
-                if (startUtc.HasValue)
-                {
-                    if (log.TimestampUtc < startUtc) return;
-                }
-
-                if (map.ContainsKey(log.Message)) map[log.Message] += 1;
-                else map.Add(log.Message, 1);
-            });
-            return map;
-        }
-    }
-}

Tests/DistTestCore/Configuration.cs

@@ -24,9 +24,6 @@ namespace DistTestCore
             this.dataFilesPath = dataFilesPath;
         }
 
-        /// <summary>
-        /// Does not override [DontDownloadLogs] attribute.
-        /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
@@ -39,7 +36,7 @@ namespace DistTestCore
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.K8sOperationRetryDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay(),
                 kubernetesNamespace: k8sNamespace
             );
 

Tests/DistTestCore/DistTest.cs

@@ -98,7 +98,7 @@ namespace DistTestCore
             }
             catch (Exception ex)
             {
-                fixtureLog.Error("Cleanup failed: " + ex);
+                fixtureLog.Error("Cleanup failed: " + ex.Message);
                 GlobalTestFailure.HasFailed = true;
             }
         }
@@ -236,19 +236,9 @@ namespace DistTestCore
         }
 
         private bool ShouldUseLongTimeouts()
-        {
-            return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();
-        }
-
-        private bool HasDontDownloadAttribute()
-        {
-            return CurrentTestMethodHasAttribute<DontDownloadLogsAttribute>();
-        }
-
-        private bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
         {
             // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out
+            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
             // if the attribute is present.
             var currentTest = TestContext.CurrentContext.Test;
             var className = currentTest.ClassName;
@@ -257,7 +247,7 @@ namespace DistTestCore
             var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
             var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
 
-            return testMethods.Any(m => m.GetCustomAttribute<T>() != null);
+            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
         }
 
         private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
@@ -278,10 +268,9 @@ namespace DistTestCore
         private bool ShouldDownloadAllLogs(TestStatus testStatus)
         {
             if (configuration.AlwaysDownloadContainerLogs) return true;
-            if (!IsDownloadingLogsEnabled()) return false;
             if (testStatus == TestStatus.Failed)
             {
-                return true;
+                return IsDownloadingLogsEnabled();
             }
 
             return false;
@@ -299,7 +288,8 @@ namespace DistTestCore
 
         private bool IsDownloadingLogsEnabled()
         {
-            return !HasDontDownloadAttribute();
+            var testProperties = TestContext.CurrentContext.Test.Properties;
+            return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
         }
     }
 

Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs

@@ -3,11 +3,11 @@
 namespace DistTestCore
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsAttribute : PropertyAttribute
+    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
     {
         public const string DontDownloadKey = "DontDownloadLogs";
 
-        public DontDownloadLogsAttribute()
+        public DontDownloadLogsOnFailureAttribute()
             : base(DontDownloadKey)
         {
         }

Tests/DistTestCore/Helpers/AssertHelpers.cs

@@ -14,7 +14,7 @@ namespace DistTestCore.Helpers
                 Time.WaitUntil(() => {
                     var c = constraint.Resolve();
                     return c.ApplyTo(actual()).IsSuccess;
-                }, "RetryAssert: " + message);
+                });
             }
             catch (TimeoutException)
             {

Tests/DistTestCore/TestLifecycle.cs

@@ -13,7 +13,7 @@ namespace DistTestCore
         private const string TestsType = "dist-tests";
         private readonly EntryPoint entryPoint;
         private readonly Dictionary<string, string> metadata; 
-        private readonly List<RunningPod> runningContainers = new();
+        private readonly List<RunningContainers> runningContainers = new();
         private readonly string deployId;
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
@@ -65,12 +65,12 @@ namespace DistTestCore
             return DateTime.UtcNow - TestStart;
         }
 
-        public void OnContainersStarted(RunningPod rc)
+        public void OnContainersStarted(RunningContainers rc)
         {
             runningContainers.Add(rc);
         }
 
-        public void OnContainersStopped(RunningPod rc)
+        public void OnContainersStopped(RunningContainers rc)
         {
             runningContainers.Remove(rc);
         }
@@ -93,20 +93,13 @@ namespace DistTestCore
 
         public void DownloadAllLogs()
         {
-            try
+            foreach (var rc in runningContainers)
             {
-                foreach (var rc in runningContainers)
+                foreach (var c in rc.Containers)
                 {
-                    foreach (var c in rc.Containers)
-                    {
-                        CoreInterface.DownloadLog(c);
-                    }
+                    CoreInterface.DownloadLog(c);
                 }
             }
-            catch (Exception ex)
-            {
-                Log.Error("Exception during log download: " + ex);
-            }
         }
     }
 }

Tests/WakuTests/BasicTests.cs

@@ -0,0 +1,27 @@
+using NUnit.Framework;
+using WakuPlugin;
+
+namespace WakuTests
+{
+    public class BasicTests : WakuDistTest
+    {
+        [Test]
+        public void Hi()
+        {
+            var bootNode = Ci.StartWakuNode(s => s.WithName("BootstrapNode"));
+            var node = Ci.StartWakuNode(s => s.WithName("Waku1").WithBootstrapNode(bootNode));
+
+            var topic = "cheeseWheels";
+            var message = "hmm, cheese...";
+
+            bootNode.SubscribeToTopic(topic);
+            node.SubscribeToTopic(topic);
+
+            node.SendMessage(topic, message);
+
+            var received = bootNode.GetMessages(topic);
+
+            CollectionAssert.Contains(received, message);
+        }
+    }
+}

Tests/WakuTests/WakuDistTest.cs

@@ -0,0 +1,13 @@
+using Core;
+using DistTestCore;
+
+namespace WakuTests
+{
+    public class WakuDistTest : DistTest
+    {
+        public WakuDistTest()
+        {
+            ProjectPlugin.Load<WakuPlugin.WakuPlugin>();
+        }
+    }
+}

Tests/WakuTests/WakuTests.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="nunit" Version="3.13.3" />
+		<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+	</ItemGroup>
+
+	<ItemGroup>
+    <ProjectReference Include="..\..\ProjectPlugins\WakuPlugin\WakuPlugin.csproj" />
+    <ProjectReference Include="..\..\Tests\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/BiblioTech/AdminChecker.cs

@@ -24,7 +24,7 @@ namespace BiblioTech
 
         public bool IsAdminChannel(IChannel channel)
         {
-            return channel.Id == Program.Config.AdminChannelId;
+            return channel.Name == Program.Config.AdminChannelName;
         }
 
         public ISocketMessageChannel GetAdminChannel()
@@ -45,7 +45,7 @@ namespace BiblioTech
         private void UpdateAdminIds()
         {
             lastUpdate = DateTime.UtcNow;
-            var adminRole = guild.Roles.Single(r => r.Id == Program.Config.AdminRoleId);
+            var adminRole = guild.Roles.Single(r => r.Name == Program.Config.AdminRoleName);
             adminIds = adminRole.Members.Select(m => m.Id).ToArray();
         }
     }

Tools/BiblioTech/CommandHandler.cs

@@ -22,7 +22,7 @@ namespace BiblioTech
 
         private async Task Client_Ready()
         {
-            var guild = client.Guilds.Single(g => g.Id == Program.Config.ServerId);
+            var guild = client.Guilds.Single(g => g.Name == Program.Config.ServerName);
             Program.AdminChecker.SetGuild(guild);
             Program.Log.Log($"Initializing for guild: '{guild.Name}'");
 

Tools/BiblioTech/Commands/MarketCommand.cs

@@ -31,8 +31,7 @@ namespace BiblioTech.Commands
 
         private string[] GetInsight(MarketAverage avg)
         {
-            var timeRange = TimeSpan.FromSeconds(avg.TimeRangeSeconds);
-            var headerLine = $"[Last {Time.FormatDuration(timeRange)}] ({avg.NumberOfFinished} Contracts finished)";
+            var headerLine = $"[Last {Time.FormatDuration(avg.TimeRange)}] ({avg.NumberOfFinished} Contracts finished)";
 
             if (avg.NumberOfFinished == 0)
             {

Tools/BiblioTech/Configuration.cs

@@ -7,31 +7,31 @@ namespace BiblioTech
         [Uniform("token", "t", "TOKEN", true, "Discord Application Token")]
         public string ApplicationToken { get; set; } = string.Empty;
 
-        [Uniform("server-id", "sn", "SERVERID", true, "ID of the Discord server")] 
-        public ulong ServerId { get; set; }
+        [Uniform("server-name", "sn", "SERVERNAME", true, "Name of the Discord server")]
+        public string ServerName { get; set; } = string.Empty;
 
-        [Uniform("datapath", "dp", "DATAPATH", true, "Root path where all data files will be saved.")]
+        [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
         
-        [Uniform("admin-role-id", "a", "ADMINROLEID", true, "ID of the Discord server admin role")]
-        public ulong AdminRoleId { get; set; }
+        [Uniform("admin-role", "a", "ADMINROLE", true, "Name of the Discord server admin role")]
+        public string AdminRoleName { get; set; } = string.Empty;
 
-        [Uniform("admin-channel-id", "ac", "ADMINCHANNELID", true, "ID of the Discord server channel where admin commands are allowed.")]
-        public ulong AdminChannelId{ get; set; }
+        [Uniform("admin-channel-name", "ac", "ADMINCHANNELNAME", true, "Name of the Discord server channel where admin commands are allowed.")]
+        public string AdminChannelName { get; set; } = "admin-channel";
 
-        [Uniform("rewards-channel-id", "rc", "REWARDSCHANNELID", false, "ID of the Discord server channel where participation rewards will be announced.")]
-        public ulong RewardsChannelId { get; set; }
+        [Uniform("rewards-channel-name", "rc", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")]
+        public string RewardsChannelName { get; set; } = "";
 
-        [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")]
-        public ulong ChainEventsChannelId { get; set; }
+        [Uniform("chain-events-channel-name", "cc", "CHAINEVENTSCHANNELNAME", false, "Name of the Discord server channel where chain events will be posted.")]
+        public string ChainEventsChannelName { get; set; } = "";
 
-        [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")]
+        [Uniform("reward-api-port", "rp", "REWARDAPIPORT", false, "TCP listen port for the reward API.")]
         public int RewardApiPort { get; set; } = 31080;
 
-        [Uniform("send-eth", "se", "SENDETH", true, "Amount of Eth send by the mint command.")]
+        [Uniform("send-eth", "se", "SENDETH", false, "Amount of Eth send by the mint command. Default: 10.")]
         public int SendEth { get; set; } = 10;
 
-        [Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TestTokens minted by the mint command.")]
+        [Uniform("mint-tt", "mt", "MINTTT", false, "Amount of TestTokens minted by the mint command. Default: 1073741824")]
         public int MintTT { get; set; } = 1073741824;
 
         public string EndpointsPath

Tools/BiblioTech/Program.cs

@@ -21,8 +21,6 @@ namespace BiblioTech
 
         public static Task Main(string[] args)
         {
-            Log = new ConsoleLog();
-
             var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
             Config = uniformArgs.Parse();
 

Tools/BiblioTech/Rewards/RewardController.cs

@@ -23,10 +23,7 @@ namespace BiblioTech.Rewards
         {
             try
             {
-                if (cmd.Averages != null && cmd.Averages.Any())
-                {
-                    Program.Averages = cmd.Averages;
-                }
+                Program.Averages = cmd.Averages;
                 await Program.RoleDriver.GiveRewards(cmd);
             }
             catch (Exception ex)

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -16,24 +16,14 @@ namespace BiblioTech.Rewards
         {
             this.client = client;
 
-            rewardsChannel = GetChannel(Program.Config.RewardsChannelId);
-            eventsChannel = GetChannel(Program.Config.ChainEventsChannelId);
+            rewardsChannel = GetChannel(Program.Config.RewardsChannelName);
+            eventsChannel = GetChannel(Program.Config.ChainEventsChannelName);
         }
 
         public async Task GiveRewards(GiveRewardsCommand rewards)
         {
             Program.Log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'");
 
-            if (rewards.Rewards.Any())
-            {
-                await ProcessRewards(rewards);
-            }
-
-            await ProcessChainEvents(rewards.EventsOverview);
-        }
-
-        private async Task ProcessRewards(GiveRewardsCommand rewards)
-        {
             var guild = GetGuild();
             // We load all role and user information first,
             // so we don't ask the server for the same info multiple times.
@@ -43,28 +33,22 @@ namespace BiblioTech.Rewards
                 rewardsChannel);
 
             await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
+            await ProcessChainEvents(rewards.EventsOverview);
         }
 
-        private SocketTextChannel? GetChannel(ulong id)
+        private SocketTextChannel? GetChannel(string name)
         {
-            if (id == 0) return null;
-            return GetGuild().TextChannels.SingleOrDefault(c => c.Id == id);
+            if (string.IsNullOrEmpty(name)) return null;
+            return GetGuild().TextChannels.SingleOrDefault(c => c.Name == name);
         }
 
         private async Task ProcessChainEvents(string[] eventsOverview)
         {
             if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
-            await Task.Run(async () =>
+            foreach (var e in eventsOverview)
             {
-                foreach (var e in eventsOverview)
-                {
-                    if (!string.IsNullOrEmpty(e))
-                    {
-                        await eventsChannel.SendMessageAsync(e);
-                        await Task.Delay(3000);
-                    }
-                }
-            });
+                await eventsChannel.SendMessageAsync(e);
+            }
         }
 
         private async Task<Dictionary<ulong, IGuildUser>> LoadAllUsers(SocketGuild guild)
@@ -147,11 +131,11 @@ namespace BiblioTech.Rewards
 
         private SocketGuild GetGuild()
         {
-            var guild = client.Guilds.SingleOrDefault(g => g.Id == Program.Config.ServerId);
+            var guild = client.Guilds.SingleOrDefault(g => g.Name == Program.Config.ServerName);
             if (guild == null)
             {
-                throw new Exception($"Unable to find guild by id: '{Program.Config.ServerId}'. " +
-                    $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name + " (" + g.Id + ")"))}]");
+                throw new Exception($"Unable to find guild by name: '{Program.Config.ServerName}'. " +
+                    $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name))}]");
             }
             return guild;
         }

Tools/CodexNetDeployer/Deployer.cs

@@ -122,7 +122,7 @@ namespace CodexNetDeployer
             });
         }
 
-        private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
+        private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
             CodexContractsDeployment contractsDeployment)
         {
             if (!config.DeployDiscordBot) return null;
@@ -155,7 +155,7 @@ namespace CodexNetDeployer
             return rc;
         }
 
-        private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
+        private RunningContainers? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
         {
             if (!config.MetricsScraper || !startResults.Any()) return null;
 
@@ -180,7 +180,7 @@ namespace CodexNetDeployer
 
         private CodexInstance CreateCodexInstance(ICodexNode node)
         {
-            return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
+            return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo());
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -255,9 +255,9 @@ namespace CodexNetDeployer
             return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan HttpRetryTimeout()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromSeconds(30);
+            return 2;
         }
 
         public TimeSpan HttpCallTimeout()
@@ -270,7 +270,7 @@ namespace CodexNetDeployer
             return TimeSpan.FromMinutes(10);
         }
 
-        public TimeSpan K8sOperationRetryDelay()
+        public TimeSpan WaitForK8sServiceDelay()
         {
             return TimeSpan.FromSeconds(30);
         }