sets quota

Framework/Core/DownloadedLog.cs

@@ -4,6 +4,7 @@ namespace Core
 {
     public interface IDownloadedLog
     {
+        void IterateLines(Action<string> action);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
@@ -18,6 +19,19 @@ namespace Core
             this.logFile = logFile;
         }
 
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
         public string[] GetLinesContaining(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);

Framework/Core/TimeSet.cs

@@ -5,7 +5,7 @@
         TimeSpan HttpCallTimeout();
         int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
+        TimeSpan K8sOperationRetryDelay();
         TimeSpan K8sOperationTimeout();
     }
 
@@ -26,7 +26,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -54,14 +54,14 @@
             return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(15);
+            return TimeSpan.FromHours(1);
         }
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -16,7 +16,7 @@
     public class MarketAverage
     {
         public int NumberOfFinished { get; set; }
-        public TimeSpan TimeRange { get; set; }
+        public int TimeRangeSeconds { get; set; }
         public float Price { get; set; }
         public float Size { get; set; }
         public float Duration { get; set; }

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,6 +43,11 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container.Recipe.Name);
+        }
+
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -372,7 +377,6 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -701,7 +705,7 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
@@ -864,7 +868,7 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
         private void WaitUntilDeploymentOnline(string deploymentName)
@@ -873,7 +877,7 @@ namespace KubernetesWorkflow
             {
                 var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            });
+            }, nameof(WaitUntilDeploymentOnline));
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -883,7 +887,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            });
+            }, nameof(WaitUntilDeploymentOffline));
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -892,19 +896,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            });
+            }, nameof(WaitUntilPodsForDeploymentAreOffline));
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
             }
             finally
             {
-                sw.End("", 1);
+                sw.End(msg, 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningContainers runningContainers);
-        void OnContainersStopped(RunningContainers runningContainers);
+        void OnContainersStarted(RunningPod runningPod);
+        void OnContainersStopped(RunningPod runningPod);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningContainers runningContainers)
+        public void OnContainersStarted(RunningPod runningPod)
         {
         }
 
-        public void OnContainersStopped(RunningContainers runningContainers)
+        public void OnContainersStopped(RunningPod runningPod)
         {
         }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,12 +9,12 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningContainers containers);
+        PodInfo GetPodInfo(RunningPod pod);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers containers, bool waitTillStopped);
+        void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,25 +60,36 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningContainers(startupConfig, startResult, containers);
+                var rc = new RunningPod(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return rc;
+                return new FutureContainers(rc, this);
+            });
+        }
+
+        public void WaitUntilOnline(RunningPod rc)
+        {
+            K8s(controller =>
+            {
+                foreach (var c in rc.Containers)
+                {
+                    controller.WaitUntilOnline(c);
+                }
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningContainers containers)
+        public PodInfo GetPodInfo(RunningPod pod)
         {
-            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -86,12 +97,12 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
+        public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
+                controller.Stop(runningPod.StartResult, waitTillStopped);
+                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow.Types
+{
+    public class FutureContainers
+    {
+        private readonly RunningPod runningPod;
+        private readonly StartupWorkflow workflow;
+
+        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
+        {
+            this.runningPod = runningPod;
+            this.workflow = workflow;
+        }
+
+        public RunningPod WaitForOnline()
+        {
+            workflow.WaitUntilOnline(runningPod);
+            return runningPod;
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
         public ContainerAddress[] Addresses { get; }
 
         [JsonIgnore]
-        public RunningContainers RunningContainers { get; internal set; } = null!;
+        public RunningPod RunningPod { get; internal set; } = null!;
 
         public Address GetAddress(ILog log, string portTag)
         {

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -2,15 +2,15 @@
 
 namespace KubernetesWorkflow.Types
 {
-    public class RunningContainers
+    public class RunningPod
     {
-        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
 
-            foreach (var c in containers) c.RunningContainers = this;
+            foreach (var c in containers) c.RunningPod = this;
         }
 
         public StartupConfig StartupConfig { get; }
@@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types
 
     public static class RunningContainersExtensions
     {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
+        public static string Describe(this RunningPod[] runningContainers)
         {
             return string.Join(",", runningContainers.Select(c => c.Describe()));
         }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,6 +18,13 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
+        public BlockTimeEntry Get(ulong blockNumber)
+        {
+            var b = cache.Get(blockNumber);
+            if (b != null) return b;
+            return GetBlock(blockNumber);
+        }
+
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -121,5 +121,12 @@ namespace NethereumWorkflow
                 to: toBlock.Value
             );
         }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+            return blockTimeFinder.Get(number);
+        }
     }
 }

Framework/Utils/Formatter.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Globalization;
+
+namespace Utils
 {
     public static class Formatter
     {
@@ -10,7 +12,7 @@
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString() + sizeSuffixes[sizeOrder];
+            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,6 +2,7 @@
 {
     public class NumberSource
     {
+        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -11,8 +12,12 @@
 
         public int GetNextNumber()
         {
-            var n = number;
-            number++;
+            var n = -1;
+            lock (@lock)
+            {
+                n = number;
+                number++;
+            }
             return n;
         }
     }

Framework/Utils/Time.cs

@@ -57,24 +57,27 @@
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate)
+        public static void WaitUntil(Func<bool> predicate, string msg)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
         {
             var start = DateTime.UtcNow;
+            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                if (DateTime.UtcNow - start > timeout)
+                var duration = DateTime.UtcNow - start;
+                if (duration > timeout)
                 {
-                    throw new TimeoutException("Operation timed out.");
+                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
+                tries++;
             }
         }
 

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -5,6 +5,9 @@ using Nethereum.ABI;
 using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -28,6 +31,7 @@ namespace CodexContractsPlugin
         SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
     }
 
+    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -86,7 +90,7 @@ namespace CodexContractsPlugin
                     {
                         var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
                         var result = requestEvent.ReturnValue1;
-                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                         result.RequestId = e.Event.RequestId;
                         return result;
                     })
@@ -99,7 +103,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
@@ -110,7 +114,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
@@ -121,7 +125,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
                 return result;
             }).ToArray();
@@ -133,7 +137,7 @@ namespace CodexContractsPlugin
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
@@ -166,6 +170,11 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            return gethNode.GetBlockForNumber(number);
+        }
+
         private EthAddress GetEthAddressFromTransaction(string transactionHash)
         {
             var transaction = gethNode.GetTransaction(transactionHash);

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,7 +59,7 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            });
+            }, nameof(DeployContract));
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@@ -71,7 +71,7 @@ namespace CodexContractsPlugin
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
             Log("Synced. Codex SmartContracts deployed.");
 
@@ -83,9 +83,9 @@ namespace CodexContractsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexContractsPlugin.Marketplace;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -53,7 +54,18 @@ namespace CodexContractsPlugin
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            return abi!.ToString(Formatting.None);
+            var byteCode = artifact["bytecode"];
+            var abiResult = abi!.ToString(Formatting.None);
+            var byteCodeResult = byteCode!.ToString(Formatting.None);
+
+            if (byteCodeResult
+                .ToLowerInvariant()
+                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
+            {
+                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
+            }
+
+            return abiResult;
         }
 
         private static string Retry(Func<string> fetch)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,5 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
+using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
@@ -7,7 +8,7 @@ namespace CodexContractsPlugin.Marketplace
     public partial class Request : RequestBase
     {
         [JsonIgnore]
-        public ulong BlockNumber { get; set; }
+        public BlockTimeEntry Block { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
@@ -16,26 +17,26 @@ namespace CodexContractsPlugin.Marketplace
     public partial class RequestFulfilledEventDTO
     {
         [JsonIgnore]
-        public ulong BlockNumber { get; set; }
+        public BlockTimeEntry Block { get; set; }
     }
 
     public partial class RequestCancelledEventDTO
     {
         [JsonIgnore]
-        public ulong BlockNumber { get; set; }
+        public BlockTimeEntry Block { get; set; }
     }
 
     public partial class SlotFilledEventDTO
     {
         [JsonIgnore]
-        public ulong BlockNumber { get; set; }
+        public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
     public partial class SlotFreedEventDTO
     {
         [JsonIgnore]
-        public ulong BlockNumber { get; set; }
+        public BlockTimeEntry Block { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -29,31 +29,31 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningContainers Deploy(DiscordBotStartupConfig config)
+        public RunningPod Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        public RunningContainers DeployRewarder(RewarderBotStartupConfig config)
+        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartRewarderContainer(workflow, config);
         }
 
-        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
         }
 
-        private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.Add(config);
-            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig);
+            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
-        public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
         {
             return Plugin(ci).DeployRewarder(config);
         }

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -38,7 +38,7 @@ namespace CodexPlugin
             if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
         }
 
-        public void CheckCompatibility(RunningContainers[] containers)
+        public void CheckCompatibility(RunningPod[] containers)
         {
             if (checkPassed) return;
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -13,7 +13,7 @@ namespace CodexPlugin
         private readonly Mapper mapper = new Mapper();
         private bool hasContainerCrashed;
 
-        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
+        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
             Container = container;
@@ -23,7 +23,7 @@ namespace CodexPlugin
             CrashWatcher.Start(this);
         }
 
-        public RunningContainer Container { get; }
+        public RunningPod Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
         public DebugInfo GetDebugInfo()
@@ -136,7 +136,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
     public class CodexDeployment
     {
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
-            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
-            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
+            RunningPod? discordBotContainer, DeploymentMetadata metadata,
             String id)
         {
             Id = id;
@@ -24,20 +24,20 @@ namespace CodexPlugin
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningContainers? PrometheusContainer { get; }
-        public RunningContainers? DiscordBotContainer { get; }
+        public RunningPod? PrometheusContainer { get; }
+        public RunningPod? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningContainers containers, DebugInfo info)
+        public CodexInstance(RunningPod pod, DebugInfo info)
         {
-            Containers = containers;
+            Pod = pod;
             Info = info;
         }
 
-        public RunningContainers Containers { get; }
+        public RunningPod Pod { get; }
         public DebugInfo Info { get; }
     }
 

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -44,7 +44,9 @@ namespace CodexPlugin
             transferSpeeds = new TransferSpeeds();
         }
 
-        public RunningContainer Container { get { return CodexAccess.Container; } }
+        public RunningPod Pod { get { return CodexAccess.Container; } }
+        
+        public RunningContainer Container { get { return Pod.Containers.Single(); } }
         public CodexAccess CodexAccess { get; }
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
@@ -56,7 +58,7 @@ namespace CodexPlugin
         {
             get
             {
-                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
             }
         }
 
@@ -71,7 +73,7 @@ namespace CodexPlugin
 
         public string GetName()
         {
-            return CodexAccess.Container.Name;
+            return Container.Name;
         }
 
         public DebugInfo GetDebugInfo()
@@ -142,11 +144,13 @@ namespace CodexPlugin
 
         public void Stop(bool waitTillStopped)
         {
-            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-                "available for codex-nodes in groups of 1.");
-
-            Group.BringOffline(waitTillStopped);
+            CrashWatcher.Stop();
+            Group.Stop(this, waitTillStopped);
+            // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+            //     "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+            //     "available for codex-nodes in groups of 1.");
+            //
+            // Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
@@ -171,7 +175,7 @@ namespace CodexPlugin
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Container);
+            var podInfo = workflow.GetPodInfo(peer.Pod);
 
             return peerInfo.Addrs.Select(a => a
                 .Replace("0.0.0.0", podInfo.Ip))

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -35,7 +35,7 @@ namespace CodexPlugin
 
         private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
+            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
             return ethAccount.EthAddress;
         }

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -15,11 +15,11 @@ namespace CodexPlugin
     {
         private readonly CodexStarter starter;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.starter = starter;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
             Version = new DebugInfoVersion();
         }
 
@@ -39,7 +39,14 @@ namespace CodexPlugin
             Containers = null!;
         }
 
-        public RunningContainers[] Containers { get; private set; }
+        public void Stop(CodexNode node, bool waitTillStopped)
+        {
+            starter.Stop(node.Pod, waitTillStopped);
+            Nodes = Nodes.Where(n => n != node).ToArray();
+            Containers = Containers.Where(c => c != node.Pod).ToArray();
+        }
+
+        public RunningPod[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
         public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
@@ -74,9 +81,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
+        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c);
+            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
             var access = new CodexAccess(tools, c, watcher);
             return factory.CreateOnlineCodexNode(access, this);
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -32,13 +32,13 @@ namespace CodexPlugin
         {
         }
 
-        public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
             return codexStarter.WrapCodexContainers(coreInterface, containers);

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -19,7 +19,7 @@ namespace CodexPlugin
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public RunningContainers[] BringOnline(CodexSetup codexSetup)
+        public RunningPod[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -34,14 +34,14 @@ namespace CodexPlugin
             {
                 var podInfo = GetPodInfo(rc);
                 var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();
 
             return containers;
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             var codexNodeFactory = new CodexNodeFactory(pluginTools);
 
@@ -65,6 +65,14 @@ namespace CodexPlugin
             Log("Stopped.");
         }
 
+        public void Stop(RunningPod pod, bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            var workflow = pluginTools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            Log("Stopped.");
+        }
+
         public string GetCodexId()
         {
             if (versionResponse != null) return versionResponse.Version;
@@ -85,24 +93,27 @@ namespace CodexPlugin
             return startupConfig;
         }
 
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
+        private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var result = new List<RunningContainers>();
+            var futureContainers = new List<FutureContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
+                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-            return result.ToArray();
+
+            return futureContainers
+                .Select(f => f.WaitForOnline())
+                .ToArray();
         }
 
-        private PodInfo GetPodInfo(RunningContainers rc)
+        private PodInfo GetPodInfo(RunningPod rc)
         {
             var workflow = pluginTools.CreateWorkflow();
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
 
@@ -119,10 +130,10 @@ namespace CodexPlugin
             return group;
         }
 
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
+        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
         {
             Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
+            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
         }
 
         private void LogSeparator()

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);
         }

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -7,9 +7,9 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
+        public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
-            Containers = containers;
+            Pod = pod;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
@@ -17,9 +17,9 @@ namespace GethPlugin
             PubKey = pubKey;
         }
 
-        public RunningContainers Containers { get; }
+        public RunningPod Pod { get; }
         [JsonIgnore]
-        public RunningContainer Container {  get { return Containers.Containers.Single(); } }
+        public RunningContainer Container {  get { return Pod.Containers.Single(); } }
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -5,6 +5,7 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
+using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -27,6 +28,7 @@ namespace GethPlugin
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
         BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
+        BlockTimeEntry GetBlockForNumber(ulong number);
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -160,6 +162,11 @@ namespace GethPlugin
             return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
         }
 
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            return StartInteraction().GetBlockForNumber(number);
+        }
+
         protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,24 +6,24 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
         {
-            return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
+            return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,15 +31,15 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
             return starter.CollectMetricsFor(scrapeTargets);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
         {
-            runningContainer = SerializeGate.Gate(runningContainer);
-            return starter.CreateAccessForTarget(runningContainer, target);
+            runningPod = SerializeGate.Gate(runningPod);
+            return starter.CreateAccessForTarget(runningPod, target);
         }
 
         public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,7 +16,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -25,16 +25,16 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig);
+            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
         {
-            var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
+            var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
         }
 

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -49,8 +49,8 @@ namespace ContinuousTests
             var start = startUtc.ToString("o");
             var end = endUtc.ToString("o");
 
-            var containerName = container.RunningContainers.StartResult.Deployment.Name;
-            var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
+            var containerName = container.RunningPod.StartResult.Deployment.Name;
+            var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
 
             //container_name : codex3-5 - deploymentName as stored in pod
             // pod_namespace : codex - continuous - nolimits - tests - 1

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -125,8 +125,8 @@ namespace ContinuousTests
             foreach (var node in nodes)
             {
                 var container = node.Container;
-                var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
-                var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
+                var deploymentName = container.RunningPod.StartResult.Deployment.Name;
+                var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
                 elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
@@ -295,13 +295,13 @@ namespace ContinuousTests
             return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
         }
 
-        private RunningContainers[] SelectRandomContainers()
+        private RunningPod[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
+            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
             if (number == -1) return containers.ToArray();
 
-            var result = new RunningContainers[number];
+            var result = new RunningPod[number];
             for (var i = 0; i < number; i++)
             {
                 result[i] = containers.PickOneRandom();

Tests/CodexContinuousTests/StartupChecker.cs

@@ -43,13 +43,13 @@ namespace ContinuousTests
             var workflow = entryPoint.Tools.CreateWorkflow();
             foreach (var instance in deployment.CodexInstances)
             {
-                foreach (var container in instance.Containers.Containers)
+                foreach (var container in instance.Pod.Containers)
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
                     log.Log(
-                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
+                        $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
@@ -92,7 +92,7 @@ namespace ContinuousTests
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
             var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -132,7 +132,7 @@ namespace CodexTests.BasicTests
 
         private const string BytesStoredMetric = "codexRepostoreBytesUsed";
 
-        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc)
+        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc)
         {
             ScopedTestFiles(() =>
             {
@@ -154,7 +154,7 @@ namespace CodexTests.BasicTests
                     var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
 
                     return high > newBytes && newBytes > low;
-                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute));
 
                 FileUtils.TrackedFile? downloadedFile = null;
                 LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId));

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -19,23 +19,24 @@ namespace CodexTests.BasicTests
 
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
+            
+            var numberOfHosts = 5;
+            var hosts = AddCodex(numberOfHosts, s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                {
+                    ContractClock = CodexLogLevel.Trace,
+                })
+                .WithStorageQuota(11.GB())
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
 
-            var numberOfHosts = 3;
-            for (var i = 0; i < numberOfHosts; i++)
+            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
+            foreach (var host in hosts)
             {
-                var host = AddCodex(s => s
-                    .WithName("Host")
-                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                    {
-                        ContractClock = CodexLogLevel.Trace,
-                    })
-                    .WithStorageQuota(11.GB())
-                    .EnableMarketplace(geth, contracts, m => m
-                        .WithInitial(10.Eth(), hostInitialBalance)
-                        .AsStorageNode()
-                        .AsValidator()));
-
-                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
+                AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
 
                 var availability = new StorageAvailability(
                     totalSpace: 10.GB(),

Tests/CodexTests/BasicTests/NetworkIsolationTest.cs

@@ -19,7 +19,7 @@ namespace CodexTests.BasicTests
         {
             node = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
+            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait));
         }
 
         [Test]
@@ -27,7 +27,7 @@ namespace CodexTests.BasicTests
         {
             var myNode = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
+            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects));
 
             try
             {

Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs

@@ -0,0 +1,83 @@
+using DistTestCore;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class ClusterDiscSpeedTests : DistTest
+    {
+        private readonly Random random = new Random();
+
+        [Test]
+        [Combinatorial]
+        public void DiscSpeedTest(
+            [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb
+        )
+        {
+            long targetSize = (long)(1024 * 1024 * 1024) * 2;
+            long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
+
+            var filename = nameof(DiscSpeedTest);
+
+            Thread.Sleep(2000);
+            if (File.Exists(filename)) File.Delete(filename);
+            Thread.Sleep(2000);
+            var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
+            Thread.Sleep(2000);
+            var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
+            
+            Log($"Write speed: {writeSpeed} per second.");
+            Log($"Read speed: {readSpeed} per second.");
+        }
+
+        private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename)
+        {
+            long bytesWritten = 0;
+            var buffer = new byte[bufferSizeBytes];
+            random.NextBytes(buffer);
+
+            var sw = Stopwatch.Begin(GetTestLog());
+            using (var stream = File.OpenWrite(filename))
+            {
+                while (bytesWritten < targetSize)
+                {
+                    long remaining = targetSize - bytesWritten;
+                    long toWrite = Math.Min(bufferSizeBytes, remaining);
+
+                    stream.Write(buffer, 0, Convert.ToInt32(toWrite));
+                    bytesWritten += toWrite;
+                }
+            }
+            var duration = sw.End("WriteTime");
+            double totalSeconds = duration.TotalSeconds;
+            double totalBytes = bytesWritten;
+            double bytesPerSecond = totalBytes / totalSeconds;
+            return new ByteSize(Convert.ToInt64(bytesPerSecond));
+        }
+
+        private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename)
+        {
+            long bytesRead = 0;
+            var buffer = new byte[bufferSizeBytes];
+            var sw = Stopwatch.Begin(GetTestLog());
+            using (var stream = File.OpenRead(filename))
+            {
+                while (bytesRead < targetSize)
+                {
+                    long remaining = targetSize - bytesRead;
+                    long toRead = Math.Min(bufferSizeBytes, remaining);
+
+                    var r = stream.Read(buffer, 0, Convert.ToInt32(toRead));
+                    bytesRead += r;
+                }
+            }
+            var duration = sw.End("ReadTime");
+            double totalSeconds = duration.TotalSeconds;
+            double totalBytes = bytesRead;
+            double bytesPerSecond = totalBytes / totalSeconds;
+            return new ByteSize(Convert.ToInt64(bytesPerSecond));
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -0,0 +1,120 @@
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class MultiPeerDownloadTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [DontDownloadLogs]
+        [UseLongTimeouts]
+        [Combinatorial]
+        public void MultiPeerDownload(
+            [Values(5, 10, 20)] int numberOfHosts,
+            [Values(100, 1000)] int fileSize
+        )
+        {
+            var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = hosts[0].UploadFile(file);
+            var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
+
+            var uploadLog = Ci.DownloadLog(hosts[0]);
+            var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
+            var blockCids = uploadLog
+                .FindLinesThatContain("Putting block into network store")
+                .Select(s =>
+                {
+                    var start = s.IndexOf("cid=") + 4;
+                    var end = s.IndexOf(" count=");
+                    var len = end - start;
+                    return s.Substring(start, len);
+                })
+                .ToArray();
+
+            Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
+
+            foreach (var h in hosts) h.DownloadContent(cid);
+
+            var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var resultFile = client.DownloadContent(cid);
+            resultFile!.AssertIsEqual(file);
+
+            var downloadLog = Ci.DownloadLog(client);
+            var host = string.Empty;
+            var blockCidHostMap = new Dictionary<string, string>();
+            downloadLog.IterateLines(line =>
+            {
+                if (line.Contains("peer=") && line.Contains(" len="))
+                {
+                    var start = line.IndexOf("peer=") + 5;
+                    var end = line.IndexOf(" len=");
+                    var len = end - start;
+                    host = line.Substring(start, len);
+                }
+                else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
+                {
+                    var start = line.IndexOf("cid=") + 4;
+                    var end = line.IndexOf(" count=");
+                    var len = end - start;
+                    var blockCid = line.Substring(start, len);
+
+                    blockCidHostMap.Add(blockCid, host);
+                    host = string.Empty;
+                }
+            });
+
+            var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
+            //PrintFullMap(blockCidHostMap);
+            PrintOverview(blockCidHostMap);
+
+            Log("Expected number of blocks: " + expectedNumberOfBlocks);
+            Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
+            Log("Total blocks fetched by hosts: " + totalFetched);
+            Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
+        }
+
+        private void PrintOverview(Dictionary<string, string> blockCidHostMap)
+        {
+            var overview = new Dictionary<string, int>();
+            foreach (var pair in blockCidHostMap)
+            {
+                if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
+                else overview[pair.Value]++;
+            }
+
+            Log("Blocks fetched per host:");
+            foreach (var pair in overview)
+            {
+                Log($"Host: {pair.Key} = {pair.Value}");
+            }
+        }
+
+        private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
+        {
+            Log("Per block, host it was fetched from:");
+            foreach (var pair in blockCidHostMap)
+            {
+                if (string.IsNullOrEmpty(pair.Value))
+                {
+                    Log($"block: {pair.Key} = Not seen");
+                }
+                else
+                {
+                    Log($"block: {pair.Key} = '{pair.Value}'");
+                }
+            }
+        }
+
+        private long RoundUp(long filesize, long blockSize)
+        {
+            double f = filesize;
+            double b = blockSize;
+
+            var result = Math.Ceiling(f / b);
+            return Convert.ToInt64(result);
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs

@@ -0,0 +1,39 @@
+using CodexPlugin;
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class OneClientLargeFileTests : CodexDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [UseLongTimeouts]
+        public void OneClientLargeFile([Values(
+            256,
+            512,
+            1024, // GB
+            2048,
+            4096,
+            8192,
+            16384,
+            32768,
+            65536,
+            131072
+        )] int sizeMb)
+        {
+            var testFile = GenerateTestFile(sizeMb.MB());
+
+            var node = AddCodex(s => s
+                .WithLogLevel(CodexLogLevel.Warn)
+                .WithStorageQuota((sizeMb + 10).MB())
+            );
+            var contentId = node.UploadFile(testFile);
+            var downloadedFile = node.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs

@@ -0,0 +1,129 @@
+using CodexPlugin;
+using DistTestCore;
+using FileUtils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests;
+
+[TestFixture]
+public class ScalabilityTests : CodexDistTest
+{
+    private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests";
+    private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests";
+
+    /// <summary>
+    /// We upload a file to node A, then download it with B.
+    /// Then we stop node A, and download again with node C.
+    /// </summary>
+    [Test]
+    [Combinatorial]
+    [UseLongTimeouts]
+    [DontDownloadLogs]
+    public void ShouldMaintainFileInNetwork(
+        [Values(10, 40, 80, 100)] int numberOfNodes,
+        [Values(100, 1000, 5000, 10000)] int fileSizeInMb,
+        [Values(true, false)] bool usePatchedImage
+    )
+    {
+        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
+
+        var logLevel = CodexLogLevel.Info;
+
+        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var nodes = AddCodex(numberOfNodes - 1, s => s
+            .WithBootstrapNode(bootstrap)
+            .WithLogLevel(logLevel)
+            .WithStorageQuota((fileSizeInMb + 50).MB())
+        ).ToList();
+
+        var uploader = nodes.PickOneRandom();
+        var downloader = nodes.PickOneRandom();
+
+        var testFile = GenerateTestFile(fileSizeInMb.MB());
+        var contentId = uploader.UploadFile(testFile);
+        var downloadedFile = downloader.DownloadContent(contentId);
+
+        downloadedFile!.AssertIsEqual(testFile);
+
+        uploader.Stop(true);
+
+        var otherDownloader = nodes.PickOneRandom();
+        downloadedFile = otherDownloader.DownloadContent(contentId);
+
+        downloadedFile!.AssertIsEqual(testFile);
+    }
+
+    /// <summary>
+    /// We upload a file to each node, to put a more wide-spread load on the network.
+    /// Then we run the same test as ShouldMaintainFileInNetwork.
+    /// </summary>
+    [Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")]
+    [Test]
+    [Combinatorial]
+    [UseLongTimeouts]
+    [DontDownloadLogs]
+    public void EveryoneGetsAFile(
+        [Values(10, 40, 80, 100)] int numberOfNodes,
+        [Values(100, 1000)] int fileSizeInMb,
+        [Values(true, false)] bool usePatchedImage
+    )
+    {
+        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
+
+        var logLevel = CodexLogLevel.Info;
+
+        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var nodes = AddCodex(numberOfNodes - 1, s => s
+            .WithBootstrapNode(bootstrap)
+            .WithLogLevel(logLevel)
+            .WithStorageQuota((fileSizeInMb + 50).MB())
+        ).ToList();
+
+        var pairTasks = nodes.Select(n =>
+        {
+            return Task.Run(() =>
+            {
+                var file = GenerateTestFile(fileSizeInMb.MB());
+                var cid = n.UploadFile(file);
+                return new NodeFilePair(n, file, cid);
+            });
+        });
+
+        var pairs = pairTasks.Select(t => Time.Wait(t)).ToList();
+
+        RunDoubleDownloadTest(
+            pairs.PickOneRandom(),
+            pairs.PickOneRandom(),
+            pairs.PickOneRandom()
+        );
+    }
+
+    private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2)
+    {
+        var expectedFile = source.File;
+        var cid = source.Cid;
+
+        var file1 = dl1.Node.DownloadContent(cid);
+        file1!.AssertIsEqual(expectedFile);
+
+        source.Node.Stop(true);
+
+        var file2 = dl2.Node.DownloadContent(cid);
+        file2!.AssertIsEqual(expectedFile);
+    }
+
+    public class NodeFilePair
+    {
+        public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid)
+        {
+            Node = node;
+            File = file;
+            Cid = cid;
+        }
+
+        public ICodexNode Node { get; }
+        public TrackedFile File { get; }
+        public ContentId Cid { get; }
+    }
+}

Tests/DistTestCore/Configuration.cs

@@ -24,6 +24,9 @@ namespace DistTestCore
             this.dataFilesPath = dataFilesPath;
         }
 
+        /// <summary>
+        /// Does not override [DontDownloadLogs] attribute.
+        /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
@@ -36,7 +39,7 @@ namespace DistTestCore
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay(),
+                retryDelay: timeSet.K8sOperationRetryDelay(),
                 kubernetesNamespace: k8sNamespace
             );
 

Tests/DistTestCore/DistTest.cs

@@ -98,7 +98,7 @@ namespace DistTestCore
             }
             catch (Exception ex)
             {
-                fixtureLog.Error("Cleanup failed: " + ex.Message);
+                fixtureLog.Error("Cleanup failed: " + ex);
                 GlobalTestFailure.HasFailed = true;
             }
         }
@@ -236,9 +236,19 @@ namespace DistTestCore
         }
 
         private bool ShouldUseLongTimeouts()
+        {
+            return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();
+        }
+
+        private bool HasDontDownloadAttribute()
+        {
+            return CurrentTestMethodHasAttribute<DontDownloadLogsAttribute>();
+        }
+
+        private bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
         {
             // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
+            // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out
             // if the attribute is present.
             var currentTest = TestContext.CurrentContext.Test;
             var className = currentTest.ClassName;
@@ -247,7 +257,7 @@ namespace DistTestCore
             var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
             var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
 
-            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
+            return testMethods.Any(m => m.GetCustomAttribute<T>() != null);
         }
 
         private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
@@ -268,9 +278,10 @@ namespace DistTestCore
         private bool ShouldDownloadAllLogs(TestStatus testStatus)
         {
             if (configuration.AlwaysDownloadContainerLogs) return true;
+            if (!IsDownloadingLogsEnabled()) return false;
             if (testStatus == TestStatus.Failed)
             {
-                return IsDownloadingLogsEnabled();
+                return true;
             }
 
             return false;
@@ -288,8 +299,7 @@ namespace DistTestCore
 
         private bool IsDownloadingLogsEnabled()
         {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
+            return !HasDontDownloadAttribute();
         }
     }
 

Tests/DistTestCore/DontDownloadLogsAttribute.cs

@@ -3,11 +3,11 @@
 namespace DistTestCore
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
+    public class DontDownloadLogsAttribute : PropertyAttribute
     {
         public const string DontDownloadKey = "DontDownloadLogs";
 
-        public DontDownloadLogsOnFailureAttribute()
+        public DontDownloadLogsAttribute()
             : base(DontDownloadKey)
         {
         }

Tests/DistTestCore/Helpers/AssertHelpers.cs

@@ -14,7 +14,7 @@ namespace DistTestCore.Helpers
                 Time.WaitUntil(() => {
                     var c = constraint.Resolve();
                     return c.ApplyTo(actual()).IsSuccess;
-                });
+                }, "RetryAssert: " + message);
             }
             catch (TimeoutException)
             {

Tests/DistTestCore/TestLifecycle.cs

@@ -13,7 +13,7 @@ namespace DistTestCore
         private const string TestsType = "dist-tests";
         private readonly EntryPoint entryPoint;
         private readonly Dictionary<string, string> metadata; 
-        private readonly List<RunningContainers> runningContainers = new();
+        private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
@@ -65,12 +65,12 @@ namespace DistTestCore
             return DateTime.UtcNow - TestStart;
         }
 
-        public void OnContainersStarted(RunningContainers rc)
+        public void OnContainersStarted(RunningPod rc)
         {
             runningContainers.Add(rc);
         }
 
-        public void OnContainersStopped(RunningContainers rc)
+        public void OnContainersStopped(RunningPod rc)
         {
             runningContainers.Remove(rc);
         }
@@ -93,13 +93,20 @@ namespace DistTestCore
 
         public void DownloadAllLogs()
         {
-            foreach (var rc in runningContainers)
+            try
             {
-                foreach (var c in rc.Containers)
+                foreach (var rc in runningContainers)
                 {
-                    CoreInterface.DownloadLog(c);
+                    foreach (var c in rc.Containers)
+                    {
+                        CoreInterface.DownloadLog(c);
+                    }
                 }
             }
+            catch (Exception ex)
+            {
+                Log.Error("Exception during log download: " + ex);
+            }
         }
     }
 }

Tools/BiblioTech/Commands/MarketCommand.cs

@@ -31,7 +31,8 @@ namespace BiblioTech.Commands
 
         private string[] GetInsight(MarketAverage avg)
         {
-            var headerLine = $"[Last {Time.FormatDuration(avg.TimeRange)}] ({avg.NumberOfFinished} Contracts finished)";
+            var timeRange = TimeSpan.FromSeconds(avg.TimeRangeSeconds);
+            var headerLine = $"[Last {Time.FormatDuration(timeRange)}] ({avg.NumberOfFinished} Contracts finished)";
 
             if (avg.NumberOfFinished == 0)
             {

Tools/BiblioTech/Rewards/RewardController.cs

@@ -23,7 +23,10 @@ namespace BiblioTech.Rewards
         {
             try
             {
-                Program.Averages = cmd.Averages;
+                if (cmd.Averages != null && cmd.Averages.Any())
+                {
+                    Program.Averages = cmd.Averages;
+                }
                 await Program.RoleDriver.GiveRewards(cmd);
             }
             catch (Exception ex)

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -24,6 +24,16 @@ namespace BiblioTech.Rewards
         {
             Program.Log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'");
 
+            if (rewards.Rewards.Any())
+            {
+                await ProcessRewards(rewards);
+            }
+
+            await ProcessChainEvents(rewards.EventsOverview);
+        }
+
+        private async Task ProcessRewards(GiveRewardsCommand rewards)
+        {
             var guild = GetGuild();
             // We load all role and user information first,
             // so we don't ask the server for the same info multiple times.
@@ -33,7 +43,6 @@ namespace BiblioTech.Rewards
                 rewardsChannel);
 
             await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
-            await ProcessChainEvents(rewards.EventsOverview);
         }
 
         private SocketTextChannel? GetChannel(string name)
@@ -45,10 +54,17 @@ namespace BiblioTech.Rewards
         private async Task ProcessChainEvents(string[] eventsOverview)
         {
             if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
-            foreach (var e in eventsOverview)
+            await Task.Run(async () =>
             {
-                await eventsChannel.SendMessageAsync(e);
-            }
+                foreach (var e in eventsOverview)
+                {
+                    if (!string.IsNullOrEmpty(e))
+                    {
+                        await eventsChannel.SendMessageAsync(e);
+                        await Task.Delay(3000);
+                    }
+                }
+            });
         }
 
         private async Task<Dictionary<ulong, IGuildUser>> LoadAllUsers(SocketGuild guild)

Tools/CodexNetDeployer/Deployer.cs

@@ -122,7 +122,7 @@ namespace CodexNetDeployer
             });
         }
 
-        private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
+        private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
             CodexContractsDeployment contractsDeployment)
         {
             if (!config.DeployDiscordBot) return null;
@@ -155,7 +155,7 @@ namespace CodexNetDeployer
             return rc;
         }
 
-        private RunningContainers? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
+        private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
         {
             if (!config.MetricsScraper || !startResults.Any()) return null;
 
@@ -180,7 +180,7 @@ namespace CodexNetDeployer
 
         private CodexInstance CreateCodexInstance(ICodexNode node)
         {
-            return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo());
+            return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -270,7 +270,7 @@ namespace CodexNetDeployer
             return TimeSpan.FromMinutes(10);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(30);
         }

Tools/CodexNetDeployer/K8sHook.cs

@@ -18,11 +18,11 @@ namespace CodexNetDeployer
             this.metadata = metadata;
         }
 
-        public void OnContainersStarted(RunningContainers rc)
+        public void OnContainersStarted(RunningPod rc)
         {
         }
 
-        public void OnContainersStopped(RunningContainers rc)
+        public void OnContainersStopped(RunningPod rc)
         {
         }
 

Tools/TestNetRewarder/BotClient.cs

@@ -1,7 +1,5 @@
-using CodexContractsPlugin.Marketplace;
-using DiscordRewards;
+using DiscordRewards;
 using Logging;
-using Newtonsoft.Json;
 using System.Net.Http.Json;
 
 namespace TestNetRewarder
@@ -26,7 +24,7 @@ namespace TestNetRewarder
 
         public async Task<bool> SendRewards(GiveRewardsCommand command)
         {
-            if (command == null || command.Rewards == null || !command.Rewards.Any()) return false;
+            if (command == null) return false;
             var result = await HttpPostJson(command);
             log.Log("Reward response: " + result);
             return result == "OK";

Tools/TestNetRewarder/ChainState.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
+using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 using Utils;
 
@@ -8,9 +9,34 @@ namespace TestNetRewarder
     public class ChainState
     {
         private readonly HistoricState historicState;
+        private readonly string[] colorIcons = new[]
+        {
+            "🔴",
+            "🟠",
+            "🟡",
+            "🟢",
+            "🔵",
+            "🟣",
+            "🟤",
+            "⚫",
+            "⚪",
+            "🟥",
+            "🟧",
+            "🟨",
+            "🟩",
+            "🟦",
+            "🟪",
+            "🟫",
+            "⬛",
+            "⬜",
+            "🔶",
+            "🔷"
+        };
 
         public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange)
         {
+            this.historicState = historicState;
+
             NewRequests = contracts.GetStorageRequests(blockRange);
             historicState.CleanUpOldRequests();
             historicState.ProcessNewRequests(NewRequests);
@@ -18,19 +44,16 @@ namespace TestNetRewarder
 
             StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray();
             FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray();
-            ChangedRequests = historicState.StorageRequests.Where(r => r.RecentlyChanged).ToArray();
             RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange);
             RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange);
             SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
             SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
-            this.historicState = historicState;
         }
 
         public Request[] NewRequests { get; }
         public StorageRequest[] AllRequests => historicState.StorageRequests;
         public StorageRequest[] StartedRequests { get; private set; }
         public StorageRequest[] FinishedRequests { get; private set; }
-        public StorageRequest[] ChangedRequests { get; private set; }
         public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; }
         public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
         public SlotFilledEventDTO[] SlotFilledEvents { get; }
@@ -40,57 +63,78 @@ namespace TestNetRewarder
         {
             var entries = new List<StringBlockNumberPair>();
 
-            entries.AddRange(ChangedRequests.Select(ToPair));
+            entries.AddRange(NewRequests.Select(ToPair));
             entries.AddRange(RequestFulfilledEvents.Select(ToPair));
             entries.AddRange(RequestCancelledEvents.Select(ToPair));
             entries.AddRange(SlotFilledEvents.Select(ToPair));
             entries.AddRange(SlotFreedEvents.Select(ToPair));
+            entries.AddRange(FinishedRequests.Select(ToPair));
 
             entries.Sort(new StringUtcComparer());
 
             return entries.Select(ToLine).ToArray();
         }
 
-        private StringBlockNumberPair ToPair(StorageRequest r)
+        private StringBlockNumberPair ToPair(Request r)
         {
-            return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.Request.BlockNumber);
+            return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        public StringBlockNumberPair ToPair(StorageRequest r)
+        {
+            return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId);
         }
 
         private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r)
         {
-            return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
+            return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
         }
 
         private StringBlockNumberPair ToPair(RequestCancelledEventDTO r)
         {
-            return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
+            return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
         }
 
         private StringBlockNumberPair ToPair(SlotFilledEventDTO r)
         {
-            return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
+            return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
         }
 
         private StringBlockNumberPair ToPair(SlotFreedEventDTO r)
         {
-            return new StringBlockNumberPair(JsonConvert.SerializeObject(r), r.BlockNumber);
+            return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
         }
 
         private string ToLine(StringBlockNumberPair pair)
         {
-            return $"[{pair.Number}] {pair.Str}";
+            var nl = Environment.NewLine;
+            var colorIcon = GetColorIcon(pair.RequestId);
+            return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" +
+                $"```json{nl}" +
+                $"{pair.Str}{nl}" +
+                $"```";
+        }
+
+        private string GetColorIcon(byte[] requestId)
+        {
+            var index = requestId[0] % colorIcons.Length;
+            return colorIcons[index];
         }
 
         public class StringBlockNumberPair
         {
-            public StringBlockNumberPair(string str, ulong number)
+            public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId)
             {
+                Name = name;
                 Str = str;
-                Number = number;
+                Block = block;
+                RequestId = requestId;
             }
 
+            public string Name { get; }
             public string Str { get; }
-            public ulong Number { get; }
+            public BlockTimeEntry Block { get; }
+            public byte[] RequestId { get; }
         }
 
         public class StringUtcComparer : IComparer<StringBlockNumberPair>
@@ -100,7 +144,7 @@ namespace TestNetRewarder
                 if (x == null && y == null) return 0;
                 if (x == null) return 1;
                 if (y == null) return -1;
-                return x.Number.CompareTo(y.Number);
+                return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber);
             }
         }
     }

Tools/TestNetRewarder/HistoricState.cs

@@ -28,8 +28,6 @@ namespace TestNetRewarder
                 r.State == RequestState.Finished ||
                 r.State == RequestState.Failed
             );
-
-            foreach (var r in storageRequests) r.IsNew = false;
         }
     }
 
@@ -39,13 +37,11 @@ namespace TestNetRewarder
         {
             Request = request;
             Hosts = Array.Empty<EthAddress>();
-            IsNew = true;
         }
 
         public Request Request { get; }
         public EthAddress[] Hosts { get; private set; }
         public RequestState State { get; private set; }
-        public bool IsNew { get; set; }
         
         [JsonIgnore]
         public bool RecentlyStarted { get; private set; }
@@ -53,9 +49,6 @@ namespace TestNetRewarder
         [JsonIgnore]
         public bool RecentlyFinished { get; private set; }
 
-        [JsonIgnore]
-        public bool RecentlyChanged { get; private set; }
-
         public void Update(ICodexContracts contracts)
         {
             var newHosts = GetHosts(contracts);
@@ -70,27 +63,10 @@ namespace TestNetRewarder
                 State == RequestState.Started &&
                 newState == RequestState.Finished;
 
-            RecentlyChanged =
-                IsNew ||
-                State != newState ||
-                HostsChanged(newHosts);
-
             State = newState;
             Hosts = newHosts;
         }
 
-        private bool HostsChanged(EthAddress[] newHosts)
-        {
-            if (newHosts.Length != Hosts.Length) return true;
-            
-            foreach (var newHost in newHosts) 
-            {
-                if (!Hosts.Contains(newHost)) return true;
-            }
-
-            return false;
-        }
-
         private EthAddress[] GetHosts(ICodexContracts contracts)
         {
             var result = new List<EthAddress>();

Tools/TestNetRewarder/MarketTracker.cs

@@ -42,6 +42,7 @@ namespace TestNetRewarder
         private ChainState[] SelectStates(int numberOfIntervals)
         {
             if (numberOfIntervals < 1) return Array.Empty<ChainState>();
+            if (numberOfIntervals > buffer.Count) return Array.Empty<ChainState>();
             return buffer.TakeLast(numberOfIntervals).ToArray();
         }
 
@@ -52,7 +53,7 @@ namespace TestNetRewarder
                 return new MarketAverage
                 {
                     NumberOfFinished = CountNumberOfFinishedRequests(states),
-                    TimeRange = GetTotalTimeRange(states),
+                    TimeRangeSeconds = GetTotalTimeRange(states),
                     Price = Average(states, s => s.Request.Ask.Reward),
                     Duration = Average(states, s => s.Request.Ask.Duration),
                     Size = Average(states, s => GetTotalSize(s.Request.Ask)),
@@ -92,12 +93,13 @@ namespace TestNetRewarder
                 }
             }
 
+            if (count < 1.0f) return 0.0f;
             return sum / count;
         }
 
-        private TimeSpan GetTotalTimeRange(ChainState[] states)
+        private int GetTotalTimeRange(ChainState[] states)
         {
-            return Program.Config.Interval * states.Length;
+            return Convert.ToInt32((Program.Config.Interval * states.Length).TotalSeconds);
         }
 
         private int CountNumberOfFinishedRequests(ChainState[] states)

Tools/TestNetRewarder/Processor.cs

@@ -67,9 +67,11 @@ namespace TestNetRewarder
             var marketAverages = GetMarketAverages(chainState);
             var eventsOverview = GenerateEventsOverview(chainState);
 
-            log.Log($"Found {outgoingRewards.Count} rewards to send. Found {marketAverages.Length} market averages.");
+            log.Log($"Found {outgoingRewards.Count} rewards. " +
+                $"Found {marketAverages.Length} market averages. " +
+                $"Found {eventsOverview.Length} events.");
 
-            if (outgoingRewards.Any())
+            if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any())
             {
                 if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview))
                 {