sets quota

# Conflicts:
#	Tests/CodexTests/BasicTests/MarketplaceTests.cs

Framework/Core/DownloadedLog.cs

@@ -4,6 +4,7 @@ namespace Core
 {
     public interface IDownloadedLog
     {
+        void IterateLines(Action<string> action);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
@@ -18,6 +19,19 @@ namespace Core
             this.logFile = logFile;
         }
 
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
         public string[] GetLinesContaining(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);

Framework/Core/Http.cs

@@ -51,7 +51,6 @@ namespace Core
 
         private string GetDescription()
         {
-            // todo: check this:
             return DebugStack.GetCallerName(skipFrames: 2);
         }
 

Framework/Core/TimeSet.cs

@@ -5,7 +5,7 @@
         TimeSpan HttpCallTimeout();
         int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
+        TimeSpan K8sOperationRetryDelay();
         TimeSpan K8sOperationTimeout();
     }
 
@@ -26,7 +26,7 @@
             return TimeSpan.FromSeconds(1);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(10);
         }
@@ -54,14 +54,14 @@
             return TimeSpan.FromSeconds(2);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(15);
+            return TimeSpan.FromHours(1);
         }
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,6 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
+        public string[] EventsOverview { get; set; } = Array.Empty<string>();
     }
 
     public class RewardUsersCommand
@@ -10,4 +12,15 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class MarketAverage
+    {
+        public int NumberOfFinished { get; set; }
+        public int TimeRangeSeconds { get; set; }
+        public float Price { get; set; }
+        public float Size { get; set; }
+        public float Duration { get; set; }
+        public float Collateral { get; set; }
+        public float ProofProbability { get; set; }
+    }
 }

Framework/DiscordRewards/RewardRepo.cs

@@ -21,11 +21,11 @@ namespace DiscordRewards
             }),
 
             // Finished a sizable slot
-            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
+            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
             {
                 Type = CheckType.FinishedSlot,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
+                MinSlotSize = 10.MB(),
+                MinDuration = TimeSpan.FromMinutes(5.0),
             }),
 
             // Posted any contract
@@ -41,12 +41,12 @@ namespace DiscordRewards
             }),
 
             // Started a sizable contract
-            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
             {
-                Type = CheckType.FinishedSlot,
+                Type = CheckType.StartedContract,
                 MinNumberOfHosts = 4,
-                MinSlotSize = 1.GB(),
-                MinDuration = TimeSpan.FromHours(24.0),
+                MinSlotSize = 10.MB(),
+                MinDuration = TimeSpan.FromMinutes(5.0),
             })
         };
     }

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,6 +43,11 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container.Recipe.Name);
+        }
+
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -372,7 +377,6 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);
@@ -701,7 +705,7 @@ namespace KubernetesWorkflow
 
         private string GetPodName(RunningContainer container)
         {
-            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
@@ -864,7 +868,7 @@ namespace KubernetesWorkflow
 
         private void WaitUntilNamespaceCreated() 
         {
-            WaitUntil(() => IsNamespaceOnline(K8sNamespace));
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
         private void WaitUntilDeploymentOnline(string deploymentName)
@@ -873,7 +877,7 @@ namespace KubernetesWorkflow
             {
                 var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
-            });
+            }, nameof(WaitUntilDeploymentOnline));
         }
 
         private void WaitUntilDeploymentOffline(string deploymentName)
@@ -883,7 +887,7 @@ namespace KubernetesWorkflow
                 var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
-            });
+            }, nameof(WaitUntilDeploymentOffline));
         }
 
         private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
@@ -892,19 +896,19 @@ namespace KubernetesWorkflow
             {
                 var pods = FindPodsByLabel(deployment.PodLabel);
                 return !pods.Any();
-            });
+            }, nameof(WaitUntilPodsForDeploymentAreOffline));
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
             var sw = Stopwatch.Begin(log, true);
             try
             {
-                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay());
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
             }
             finally
             {
-                sw.End("", 1);
+                sw.End(msg, 1);
             }
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -5,18 +5,18 @@ namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {
-        void OnContainersStarted(RunningContainers runningContainers);
-        void OnContainersStopped(RunningContainers runningContainers);
+        void OnContainersStarted(RunningPod runningPod);
+        void OnContainersStopped(RunningPod runningPod);
         void OnContainerRecipeCreated(ContainerRecipe recipe);
     }
 
     public class DoNothingK8sHooks : IK8sHooks
     {
-        public void OnContainersStarted(RunningContainers runningContainers)
+        public void OnContainersStarted(RunningPod runningPod)
         {
         }
 
-        public void OnContainersStopped(RunningContainers runningContainers)
+        public void OnContainersStopped(RunningPod runningPod)
         {
         }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,12 +9,12 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
-        PodInfo GetPodInfo(RunningContainers containers);
+        PodInfo GetPodInfo(RunningPod pod);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers containers, bool waitTillStopped);
+        void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -60,25 +60,36 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningContainers(startupConfig, startResult, containers);
+                var rc = new RunningPod(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
                 {
                     componentFactory.Update(controller);
                 }
-                return rc;
+                return new FutureContainers(rc, this);
+            });
+        }
+
+        public void WaitUntilOnline(RunningPod rc)
+        {
+            K8s(controller =>
+            {
+                foreach (var c in rc.Containers)
+                {
+                    controller.WaitUntilOnline(c);
+                }
             });
         }
 
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
         }
 
-        public PodInfo GetPodInfo(RunningContainers containers)
+        public PodInfo GetPodInfo(RunningPod pod)
         {
-            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -86,12 +97,12 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
+        public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
+                controller.Stop(runningPod.StartResult, waitTillStopped);
+                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow.Types
+{
+    public class FutureContainers
+    {
+        private readonly RunningPod runningPod;
+        private readonly StartupWorkflow workflow;
+
+        public FutureContainers(RunningPod runningPod, StartupWorkflow workflow)
+        {
+            this.runningPod = runningPod;
+            this.workflow = workflow;
+        }
+
+        public RunningPod WaitForOnline()
+        {
+            workflow.WaitUntilOnline(runningPod);
+            return runningPod;
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types
         public ContainerAddress[] Addresses { get; }
 
         [JsonIgnore]
-        public RunningContainers RunningContainers { get; internal set; } = null!;
+        public RunningPod RunningPod { get; internal set; } = null!;
 
         public Address GetAddress(ILog log, string portTag)
         {

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -2,15 +2,15 @@
 
 namespace KubernetesWorkflow.Types
 {
-    public class RunningContainers
+    public class RunningPod
     {
-        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
 
-            foreach (var c in containers) c.RunningContainers = this;
+            foreach (var c in containers) c.RunningPod = this;
         }
 
         public StartupConfig StartupConfig { get; }
@@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types
 
     public static class RunningContainersExtensions
     {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
+        public static string Describe(this RunningPod[] runningContainers)
         {
             return string.Join(",", runningContainers.Select(c => c.Describe()));
         }

Framework/NethereumWorkflow/BlockUtils/BlockCache.cs

@@ -35,5 +35,7 @@
             if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null;
             return value;
         }
+
+        public int Size { get { return entries.Count; } }
     }
 }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -18,13 +18,20 @@ namespace NethereumWorkflow.BlockUtils
             bounds = new BlockchainBounds(cache, web3);
         }
 
+        public BlockTimeEntry Get(ulong blockNumber)
+        {
+            var b = cache.Get(blockNumber);
+            if (b != null) return b;
+            return GetBlock(blockNumber);
+        }
+
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
             if (moment <= bounds.Genesis.Utc) return null;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector);
+            return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
         }
 
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
@@ -33,7 +40,16 @@ namespace NethereumWorkflow.BlockUtils
             if (moment >= bounds.Current.Utc) return null;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
-            return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector);
+            return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;
+        }
+
+        private ulong Log(Func<ulong> operation)
+        {
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder));
+            var result = operation();
+            sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
+
+            return result;
         }
 
         private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func<DateTime, BlockTimeEntry, bool> isWhatIwant)
@@ -70,7 +86,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var next = GetBlock(entry.BlockNumber + 1);
             return
-                entry.Utc < target &&
+                entry.Utc <= target &&
                 next.Utc > target;
         }
 
@@ -78,7 +94,7 @@ namespace NethereumWorkflow.BlockUtils
         {
             var previous = GetBlock(entry.BlockNumber - 1);
             return
-                entry.Utc > target &&
+                entry.Utc >= target &&
                 previous.Utc < target;
         }
 

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -89,26 +89,9 @@ namespace NethereumWorkflow
             }
         }
 
-        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new()
         {
-            var wrapper = new Web3Wrapper(web3, log);
-            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
-
-            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
-            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
-
-            if (!fromBlock.HasValue)
-            {
-                log.Error("Failed to find lowest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-            if (!toBlock.HasValue)
-            {
-                log.Error("Failed to find highest block for time range: " + timeRange);
-                throw new Exception("Failed");
-            }
-
-            return GetEvents<TEvent>(address, fromBlock.Value, toBlock.Value);
+            return GetEvents<TEvent>(address, blockRange.From, blockRange.To);
         }
 
         public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
@@ -119,5 +102,31 @@ namespace NethereumWorkflow
             var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
             return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
         }
+
+        public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            if (fromBlock == null  || toBlock == null)
+            {
+                throw new Exception("Failed to convert time range to block range.");
+            }
+
+            return new BlockInterval(
+                from: fromBlock.Value,
+                to: toBlock.Value
+            );
+        }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+            return blockTimeFinder.Get(number);
+        }
     }
 }

Framework/Utils/BlockInterval.cs

@@ -0,0 +1,27 @@
+namespace Utils
+{
+    public class BlockInterval
+    {
+        public BlockInterval(ulong from, ulong to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+        }
+
+        public ulong From { get; }
+        public ulong To { get; }
+
+        public override string ToString()
+        {
+            return $"[{From} - {To}]";
+        }
+    }
+}

Framework/Utils/Formatter.cs

@@ -1,4 +1,6 @@
-namespace Utils
+using System.Globalization;
+
+namespace Utils
 {
     public static class Formatter
     {
@@ -10,7 +12,7 @@
 
             var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1);
-            return digit.ToString() + sizeSuffixes[sizeOrder];
+            return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder];
         }
     }
 }

Framework/Utils/NumberSource.cs

@@ -2,6 +2,7 @@
 {
     public class NumberSource
     {
+        private readonly object @lock = new object();
         private int number;
 
         public NumberSource(int start)
@@ -11,8 +12,12 @@
 
         public int GetNextNumber()
         {
-            var n = number;
-            number++;
+            var n = -1;
+            lock (@lock)
+            {
+                n = number;
+                number++;
+            }
             return n;
         }
     }

Framework/Utils/Time.cs

@@ -57,24 +57,27 @@
             return result;
         }
 
-        public static void WaitUntil(Func<bool> predicate)
+        public static void WaitUntil(Func<bool> predicate, string msg)
         {
-            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg);
         }
 
-        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay)
+        public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryDelay, string msg)
         {
             var start = DateTime.UtcNow;
+            var tries = 1;
             var state = predicate();
             while (!state)
             {
-                if (DateTime.UtcNow - start > timeout)
+                var duration = DateTime.UtcNow - start;
+                if (duration > timeout)
                 {
-                    throw new TimeoutException("Operation timed out.");
+                    throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'");
                 }
 
                 Sleep(retryDelay);
                 state = predicate();
+                tries++;
             }
         }
 

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -5,6 +5,9 @@ using Nethereum.ABI;
 using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -19,15 +22,16 @@ namespace CodexContractsPlugin
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
 
-        Request[] GetStorageRequests(TimeRange timeRange);
+        Request[] GetStorageRequests(BlockInterval blockRange);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
-        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
-        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
-        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
-        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
     }
 
+    [JsonConverter(typeof(StringEnumConverter))]
     public enum RequestState
     {
         New,
@@ -77,63 +81,63 @@ namespace CodexContractsPlugin
             return balance.TestTokens();
         }
 
-        public Request[] GetStorageRequests(TimeRange timeRange)
+        public Request[] GetStorageRequests(BlockInterval blockRange)
         {
-            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, blockRange);
             var i = StartInteraction();
             return events
                     .Select(e =>
                     {
                         var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
                         var result = requestEvent.ReturnValue1;
-                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                         result.RequestId = e.Event.RequestId;
                         return result;
                     })
                     .ToArray();
         }
 
-        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange)
         {
-            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
 
-        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange)
         {
-            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, blockRange);
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
 
-        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
+        public SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange)
         {
-            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
                 return result;
             }).ToArray();
         }
 
-        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
+        public SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange)
         {
-            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, blockRange);
             return events.Select(e =>
             {
                 var result = e.Event;
-                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
                 return result;
             }).ToArray();
         }
@@ -166,6 +170,11 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            return gethNode.GetBlockForNumber(number);
+        }
+
         private EthAddress GetEthAddressFromTransaction(string transactionHash)
         {
             var transaction = gethNode.GetTransaction(transactionHash);

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 
@@ -59,7 +59,7 @@ namespace CodexContractsPlugin
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
                 workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
-            });
+            }, nameof(DeployContract));
             Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
@@ -71,7 +71,7 @@ namespace CodexContractsPlugin
 
             Log("Extract completed. Checking sync...");
 
-            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi));
+            Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
             Log("Synced. Codex SmartContracts deployed.");
 
@@ -83,9 +83,9 @@ namespace CodexContractsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private void WaitUntil(Func<bool> predicate)
+        private void WaitUntil(Func<bool> predicate, string msg)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg);
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexContractsPlugin.Marketplace;
+using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
@@ -53,7 +54,18 @@ namespace CodexContractsPlugin
 
             var artifact = JObject.Parse(json);
             var abi = artifact["abi"];
-            return abi!.ToString(Formatting.None);
+            var byteCode = artifact["bytecode"];
+            var abiResult = abi!.ToString(Formatting.None);
+            var byteCodeResult = byteCode!.ToString(Formatting.None);
+
+            if (byteCodeResult
+                .ToLowerInvariant()
+                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
+            {
+                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
+            }
+
+            return abiResult;
         }
 
         private static string Retry(Func<string> fetch)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,11 +1,14 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using GethPlugin;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
 {
     public partial class Request : RequestBase
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
@@ -13,23 +16,27 @@ namespace CodexContractsPlugin.Marketplace
 
     public partial class RequestFulfilledEventDTO
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
     public partial class RequestCancelledEventDTO
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 
     public partial class SlotFilledEventDTO
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
     public partial class SlotFreedEventDTO
     {
-        public ulong BlockNumber { get; set; }
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
     }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -29,18 +29,31 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningContainers Deploy(DiscordBotStartupConfig config)
+        public RunningPod Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        public RunningPod DeployRewarder(RewarderBotStartupConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            return StartRewarderContainer(workflow, config);
+        }
+
+        private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+        }
+
+        private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.Add(config);
+            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -5,11 +5,16 @@ namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }
 
+        public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config)
+        {
+            return Plugin(ci).DeployRewarder(config);
+        }
+
         private static CodexDiscordBotPlugin Plugin(CoreInterface ci)
         {
             return ci.GetPlugin<CodexDiscordBotPlugin>();

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,9 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "thatbenbierens/codex-discordbot:initial";
+        public override string Image => "codexstorage/codex-discordbot:sha-8c64352";
+
+        public static string RewardsPort = "bot_rewards_port";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -19,6 +21,7 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);
             AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("REWARDSCHANNELNAME", config.RewardChannelName);
             AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
             AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
 
@@ -30,13 +33,13 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
             AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
+            AddInternalPortAndVar("REWARDAPIPORT", RewardsPort);
+
             if (!string.IsNullOrEmpty(config.DataPath))
             {
                 AddEnvVar("DATAPATH", config.DataPath);
                 AddVolume(config.DataPath, 1.GB());
             }
-
-            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -2,7 +2,7 @@
 {
     public class DiscordBotStartupConfig
     {
-        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo, string rewardChannelName)
         {
             Name = name;
             Token = token;
@@ -11,6 +11,7 @@
             AdminChannelName = adminChannelName;
             KubeNamespace = kubeNamespace;
             GethInfo = gethInfo;
+            RewardChannelName = rewardChannelName;
         }
 
         public string Name { get; }
@@ -18,11 +19,32 @@
         public string ServerName { get; }
         public string AdminRoleName { get; }
         public string AdminChannelName { get; }
+        public string RewardChannelName { get; }
         public string KubeNamespace { get; }
         public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }
     }
 
+    public class RewarderBotStartupConfig
+    {
+        public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
+        {
+            DiscordBotHost = discordBotHost;
+            DiscordBotPort = discordBotPort;
+            IntervalMinutes = intervalMinutes;
+            HistoryStartUtc = historyStartUtc;
+            GethInfo = gethInfo;
+            DataPath = dataPath;
+        }
+
+        public string DiscordBotHost { get; }
+        public int DiscordBotPort { get; }
+        public string IntervalMinutes { get; }
+        public DateTime HistoryStartUtc { get; }
+        public DiscordBotGethInfo GethInfo { get; }
+        public string? DataPath { get; set; }
+    }
+
     public class DiscordBotGethInfo
     {
         public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -0,0 +1,39 @@
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow;
+using Utils;
+
+namespace CodexDiscordBotPlugin
+{
+    public class RewarderBotContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "discordbot-rewarder";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<RewarderBotStartupConfig>();
+
+            SetSchedulingAffinity(notIn: "false");
+
+            AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost);
+            AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString());
+            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes);
+            var offset = new DateTimeOffset(config.HistoryStartUtc);
+            AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString());
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
+
+            if (!string.IsNullOrEmpty(config.DataPath))
+            {
+                AddEnvVar("DATAPATH", config.DataPath);
+                AddVolume(config.DataPath, 1.GB());
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "DC-90-1B-63-76-1B-92-01-05-65-33-DA-17-C2-34-83-E1-2E-6C-A9-04-4D-68-ED-96-43-F5-E5-6A-00-0F-5F";
+        private const string OpenApiYamlHash = "5A-B0-2A-AC-42-B1-A2-49-6F-9D-4E-D8-56-40-10-A6-67-F4-0D-2A-9F-E0-84-5C-EB-B8-2D-4F-D8-56-79-6C";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -38,7 +38,7 @@ namespace CodexPlugin
             if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
         }
 
-        public void CheckCompatibility(RunningContainers[] containers)
+        public void CheckCompatibility(RunningPod[] containers)
         {
             if (checkPassed) return;
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -2,6 +2,7 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
@@ -12,7 +13,7 @@ namespace CodexPlugin
         private readonly Mapper mapper = new Mapper();
         private bool hasContainerCrashed;
 
-        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
+        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
             Container = container;
@@ -22,7 +23,7 @@ namespace CodexPlugin
             CrashWatcher.Start(this);
         }
 
-        public RunningContainer Container { get; }
+        public RunningPod Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
         public DebugInfo GetDebugInfo()
@@ -90,7 +91,16 @@ namespace CodexPlugin
 
         public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
+            var endpoint = GetEndpoint();
+            return Time.Retry(() =>
+            {
+                var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+            }, nameof(GetPurchaseStatus));
+
+            // TODO: current getpurchase api does not line up with its openapi spec.
+            // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
         }
 
         public string GetName()
@@ -126,7 +136,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-cd280d4-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1524803-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
     public class CodexDeployment
     {
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
-            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
-            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
+            RunningPod? discordBotContainer, DeploymentMetadata metadata,
             String id)
         {
             Id = id;
@@ -24,20 +24,20 @@ namespace CodexPlugin
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningContainers? PrometheusContainer { get; }
-        public RunningContainers? DiscordBotContainer { get; }
+        public RunningPod? PrometheusContainer { get; }
+        public RunningPod? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningContainers containers, DebugInfo info)
+        public CodexInstance(RunningPod pod, DebugInfo info)
         {
-            Containers = containers;
+            Pod = pod;
             Info = info;
         }
 
-        public RunningContainers Containers { get; }
+        public RunningPod Pod { get; }
         public DebugInfo Info { get; }
     }
 

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -44,7 +44,9 @@ namespace CodexPlugin
             transferSpeeds = new TransferSpeeds();
         }
 
-        public RunningContainer Container { get { return CodexAccess.Container; } }
+        public RunningPod Pod { get { return CodexAccess.Container; } }
+        
+        public RunningContainer Container { get { return Pod.Containers.Single(); } }
         public CodexAccess CodexAccess { get; }
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
@@ -56,7 +58,7 @@ namespace CodexPlugin
         {
             get
             {
-                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
             }
         }
 
@@ -71,7 +73,7 @@ namespace CodexPlugin
 
         public string GetName()
         {
-            return CodexAccess.Container.Name;
+            return Container.Name;
         }
 
         public DebugInfo GetDebugInfo()
@@ -142,11 +144,13 @@ namespace CodexPlugin
 
         public void Stop(bool waitTillStopped)
         {
-            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-                "available for codex-nodes in groups of 1.");
-
-            Group.BringOffline(waitTillStopped);
+            CrashWatcher.Stop();
+            Group.Stop(this, waitTillStopped);
+            // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+            //     "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+            //     "available for codex-nodes in groups of 1.");
+            //
+            // Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
@@ -171,7 +175,7 @@ namespace CodexPlugin
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Container);
+            var podInfo = workflow.GetPodInfo(peer.Pod);
 
             return peerInfo.Addrs.Select(a => a
                 .Replace("0.0.0.0", podInfo.Ip))

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -35,7 +35,7 @@ namespace CodexPlugin
 
         private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
+            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
             return ethAccount.EthAddress;
         }

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -15,11 +15,11 @@ namespace CodexPlugin
     {
         private readonly CodexStarter starter;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.starter = starter;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
             Version = new DebugInfoVersion();
         }
 
@@ -39,7 +39,14 @@ namespace CodexPlugin
             Containers = null!;
         }
 
-        public RunningContainers[] Containers { get; private set; }
+        public void Stop(CodexNode node, bool waitTillStopped)
+        {
+            starter.Stop(node.Pod, waitTillStopped);
+            Nodes = Nodes.Where(n => n != node).ToArray();
+            Containers = Containers.Where(c => c != node.Pod).ToArray();
+        }
+
+        public RunningPod[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
         public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
@@ -74,9 +81,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
+        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c);
+            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
             var access = new CodexAccess(tools, c, watcher);
             return factory.CreateOnlineCodexNode(access, this);
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -32,13 +32,13 @@ namespace CodexPlugin
         {
         }
 
-        public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
             return codexStarter.WrapCodexContainers(coreInterface, containers);

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -50,6 +50,7 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn;
         public CodexLogLevel? BlockExchange { get; }
     }
 

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -19,7 +19,7 @@ namespace CodexPlugin
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public RunningContainers[] BringOnline(CodexSetup codexSetup)
+        public RunningPod[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -34,14 +34,14 @@ namespace CodexPlugin
             {
                 var podInfo = GetPodInfo(rc);
                 var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
-                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();
 
             return containers;
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
+        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
         {
             var codexNodeFactory = new CodexNodeFactory(pluginTools);
 
@@ -65,6 +65,14 @@ namespace CodexPlugin
             Log("Stopped.");
         }
 
+        public void Stop(RunningPod pod, bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            var workflow = pluginTools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            Log("Stopped.");
+        }
+
         public string GetCodexId()
         {
             if (versionResponse != null) return versionResponse.Version;
@@ -85,24 +93,27 @@ namespace CodexPlugin
             return startupConfig;
         }
 
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
+        private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var result = new List<RunningContainers>();
+            var futureContainers = new List<FutureContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
+                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-            return result.ToArray();
+
+            return futureContainers
+                .Select(f => f.WaitForOnline())
+                .ToArray();
         }
 
-        private PodInfo GetPodInfo(RunningContainers rc)
+        private PodInfo GetPodInfo(RunningPod rc)
         {
             var workflow = pluginTools.CreateWorkflow();
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
 
@@ -119,10 +130,10 @@ namespace CodexPlugin
             return group;
         }
 
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
+        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
         {
             Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
+            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
         }
 
         private void LogSeparator()

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -77,8 +77,7 @@ namespace CodexPlugin
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
-                    // Contract clock is always set to warn. It logs a trace every second.
-                    $"{CodexLogLevel.Warn.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
+                    $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
 
                 if (CustomTopics.BlockExchange != null)
                 {

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -5,12 +5,12 @@ namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);
         }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -57,8 +57,8 @@ namespace CodexPlugin
                 Reward = ToDecInt(purchase.PricePerSlotPerSecond),
                 Collateral = ToDecInt(purchase.RequiredCollateral),
                 Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds),
-                Nodes = purchase.MinRequiredNumberOfNodes,
-                Tolerance = purchase.NodeFailureTolerance
+                Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
+                Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };
         }
 

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -65,11 +65,27 @@ components:
       description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
       default: 10 minutes
 
+    SPR:
+      type: string
+      description: Signed Peer Record (libp2p)
+
+    SPRRead:
+      type: object
+      properties:
+        spr:
+          $ref: "#/components/schemas/SPR"
+
+    PeerIdRead:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/PeerId"
+
     ErasureParameters:
       type: object
       properties:
         totalChunks:
-          type: number
+          type: integer
 
     PoRParameters:
       description: Parameters for Proof of Retrievability
@@ -106,8 +122,7 @@ components:
           type: string
           description: Path of the data repository where all nodes data are stored
         spr:
-          type: string
-          description: Signed Peer Record to advertise DHT connection information
+          $ref: "#/components/schemas/SPR"
 
     SalesAvailability:
       type: object
@@ -186,12 +201,12 @@ components:
         proofProbability:
           $ref: "#/components/schemas/ProofProbability"
         nodes:
-          type: number
           description: Minimal number of nodes the content should be stored on
+          type: integer
           default: 1
         tolerance:
-          type: number
           description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
+          type: integer
           default: 0
         collateral:
           type: string
@@ -206,8 +221,8 @@ components:
         - reward
       properties:
         slots:
-          type: number
           description: Number of slots (eq. hosts) that the Request want to have the content spread over
+          type: integer
         slotSize:
           type: string
           description: Amount of storage per slot (in bytes) as decimal string
@@ -218,7 +233,7 @@ components:
         reward:
           $ref: "#/components/schemas/Reward"
         maxSlotLoss:
-          type: number
+          type: integer
           description: Max slots that can be lost without data considered to be lost
 
     StorageRequest:
@@ -274,10 +289,10 @@ components:
           $ref: "#/components/schemas/Cid"
           description: "Root hash of the content"
         originalBytes:
-          type: number
+          type: integer
           description: "Length of original content in bytes"
         blockSize:
-          type: number
+          type: integer
           description: "Size of blocks"
         protected:
           type: boolean
@@ -287,16 +302,16 @@ components:
       type: object
       properties:
         totalBlocks:
-          type: number
           description: "Number of blocks stored by the node"
+          type: integer
         quotaMaxBytes:
-          type: number
+          type: integer
           description: "Maximum storage space used by the node"
         quotaUsedBytes:
-          type: number
+          type: integer
           description: "Amount of storage space currently in use"
         quotaReservedBytes:
-          type: number
+          type: integer
           description: "Amount of storage space reserved"
 
 servers:
@@ -685,6 +700,40 @@ paths:
         "503":
           description: Purchasing is unavailable
 
+  "/node/spr":
+    get:
+      summary: "Get Node's SPR"
+      operationId: getSPR
+      tags: [ Node ]
+      responses:
+        "200":
+          description: Node's SPR
+          content:
+            plain/text:
+              schema:
+                $ref: "#/components/schemas/SPR"
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SPRRead"
+        "503":
+          description: Node SPR not ready, try again later
+
+  "/node/peerid":
+    get:
+      summary: "Get Node's PeerID"
+      operationId: getPeerId
+      tags: [ Node ]
+      responses:
+        "200":
+          description: Node's Peer ID
+          content:
+            plain/text:
+              schema:
+                $ref: "#/components/schemas/PeerId"
+            application/json:
+              schema:
+                $ref: "#/components/schemas/PeerIdRead"
+
   "/debug/chronicles/loglevel":
     post:
       summary: "Set log level at run time"

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -4,6 +4,7 @@ using System.Text;
 public static class Program
 {
     private const string OpenApiFile = "../CodexPlugin/openapi.yaml";
+    private const string ClientFile = "../CodexPlugin/obj/openapiClient.cs";
     private const string Search = "<INSERT-OPENAPI-YAML-HASH>";
     private const string TargetFile = "ApiChecker.cs";
 
@@ -11,6 +12,9 @@ public static class Program
     {
         Console.WriteLine("Injecting hash of 'openapi.yaml'...");
 
+        // Force client rebuild by deleting previous artifact.
+        File.Delete(ClientFile);
+
         var hash = CreateHash();
         // This hash is used to verify that the Codex docker image being used is compatible
         // with the openapi.yaml being used by the Codex plugin.

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -7,9 +7,9 @@ namespace GethPlugin
 {
     public class GethDeployment : IHasContainer
     {
-        public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
+        public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey)
         {
-            Containers = containers;
+            Pod = pod;
             DiscoveryPort = discoveryPort;
             HttpPort = httpPort;
             WsPort = wsPort;
@@ -17,9 +17,9 @@ namespace GethPlugin
             PubKey = pubKey;
         }
 
-        public RunningContainers Containers { get; }
+        public RunningPod Pod { get; }
         [JsonIgnore]
-        public RunningContainer Container {  get { return Containers.Containers.Single(); } }
+        public RunningContainer Container {  get { return Pod.Containers.Single(); } }
         public Port DiscoveryPort { get; }
         public Port HttpPort { get; }
         public Port WsPort { get; }

ProjectPlugins/GethPlugin/GethNode.cs

@@ -5,6 +5,7 @@ using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
+using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -24,8 +25,10 @@ namespace GethPlugin
         decimal? GetSyncedBlockNumber();
         bool IsContractAvailable(string abi, string contractAddress);
         GethBootstrapNode GetBootstrapRecord();
-        List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new();
         List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
+        BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange);
+        BlockTimeEntry GetBlockForNumber(ulong number);
     }
 
     public class DeploymentGethNode : BaseGethNode, IGethNode
@@ -144,14 +147,24 @@ namespace GethPlugin
             return StartInteraction().IsContractAvailable(abi, contractAddress);
         }
 
-        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, BlockInterval blockRange) where TEvent : IEventDTO, new()
         {
-            return StartInteraction().GetEvents<TEvent>(address, fromBlockNumber, toBlockNumber);
+            return StartInteraction().GetEvents<TEvent>(address, blockRange);
         }
 
         public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
         {
-            return StartInteraction().GetEvents<TEvent>(address, timeRange);
+            return StartInteraction().GetEvents<TEvent>(address, ConvertTimeRangeToBlockRange(timeRange));
+        }
+
+        public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange)
+        {
+            return StartInteraction().ConvertTimeRangeToBlockRange(timeRange);
+        }
+
+        public BlockTimeEntry GetBlockForNumber(ulong number)
+        {
+            return StartInteraction().GetBlockForNumber(number);
         }
 
         protected abstract NethereumInteraction StartInteraction();

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,24 +6,24 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
         {
-            return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
+            return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,15 +31,15 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
             return starter.CollectMetricsFor(scrapeTargets);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
         {
-            runningContainer = SerializeGate.Gate(runningContainer);
-            return starter.CreateAccessForTarget(runningContainer, target);
+            runningPod = SerializeGate.Gate(runningPod);
+            return starter.CreateAccessForTarget(runningPod, target);
         }
 
         public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,7 +16,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -25,16 +25,16 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig);
+            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
         {
-            var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single());
+            var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
         }
 

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -34,11 +34,11 @@ namespace ContinuousTests
                 $"{startUtc.ToString("o")} - {endUtc.ToString("o")}");
             log.Log(openingLine);
 
-            var http = CreateElasticSearchHttp();
+            var endpoint = CreateElasticSearchEndpoint();
             var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc);
 
             targetFile.Write($"Downloading '{container.Name}' to '{targetFile.FullFilename}'.");
-            var reconstructor = new LogReconstructor(targetFile, http, queryTemplate);
+            var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
             reconstructor.DownloadFullLog();
 
             log.Log("Log download finished.");
@@ -49,8 +49,8 @@ namespace ContinuousTests
             var start = startUtc.ToString("o");
             var end = endUtc.ToString("o");
 
-            var containerName = container.RunningContainers.StartResult.Deployment.Name;
-            var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
+            var containerName = container.RunningPod.StartResult.Deployment.Name;
+            var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
 
             //container_name : codex3-5 - deploymentName as stored in pod
             // pod_namespace : codex - continuous - nolimits - tests - 1
@@ -64,34 +64,36 @@ namespace ContinuousTests
                 .Replace("<NAMESPACENAME>", namespaceName);
         }
 
-        private IHttp CreateElasticSearchHttp()
+        private IEndpoint CreateElasticSearchEndpoint()
         {
             var serviceName = "elasticsearch";
             var k8sNamespace = "monitoring";
             var address = new Address($"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
             var baseUrl = "";
 
-            return tools.CreateHttp(address, baseUrl, client =>
+            var http = tools.CreateHttp(client =>
             {
                 client.DefaultRequestHeaders.Add("kbn-xsrf", "reporting");
             });
+
+            return http.CreateEndpoint(address, baseUrl);
         }
 
         public class LogReconstructor
         {
             private readonly List<LogQueueEntry> queue = new List<LogQueueEntry>();
             private readonly LogFile targetFile;
-            private readonly IHttp http;
+            private readonly IEndpoint endpoint;
             private readonly string queryTemplate;
             private const int sizeOfPage = 2000;
             private string searchAfter = "";
             private int lastHits = 1;
             private ulong? lastLogLine;
 
-            public LogReconstructor(LogFile targetFile, IHttp http, string queryTemplate)
+            public LogReconstructor(LogFile targetFile, IEndpoint endpoint, string queryTemplate)
             {
                 this.targetFile = targetFile;
-                this.http = http;
+                this.endpoint = endpoint;
                 this.queryTemplate = queryTemplate;
             }
 
@@ -110,7 +112,7 @@ namespace ContinuousTests
                                 .Replace("<SIZE>", sizeOfPage.ToString())
                                 .Replace("<SEARCHAFTER>", searchAfter);
 
-                var response = http.HttpPostString<SearchResponse>("_search", query);
+                var response = endpoint.HttpPostString<SearchResponse>("_search", query);
 
                 lastHits = response.hits.hits.Length;
                 if (lastHits > 0)

Tests/CodexContinuousTests/NodeRunner.cs

@@ -44,7 +44,7 @@ namespace ContinuousTests
             try
             {
                 var debugInfo = bootstrapNode.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+                Assert.That(!string.IsNullOrEmpty(debugInfo.Spr));
 
                 var node = entryPoint.CreateInterface().StartCodexNode(s =>
                 {

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -125,10 +125,10 @@ namespace ContinuousTests
             foreach (var node in nodes)
             {
                 var container = node.Container;
-                var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
-                var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
+                var deploymentName = container.RunningPod.StartResult.Deployment.Name;
+                var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
-                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
+                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
                 elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
                     effectiveEnd, openingLine);
             }
@@ -295,13 +295,13 @@ namespace ContinuousTests
             return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
         }
 
-        private RunningContainers[] SelectRandomContainers()
+        private RunningPod[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
+            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
             if (number == -1) return containers.ToArray();
 
-            var result = new RunningContainers[number];
+            var result = new RunningPod[number];
             for (var i = 0; i < number; i++)
             {
                 result[i] = containers.PickOneRandom();

Tests/CodexContinuousTests/StartupChecker.cs

@@ -43,13 +43,13 @@ namespace ContinuousTests
             var workflow = entryPoint.Tools.CreateWorkflow();
             foreach (var instance in deployment.CodexInstances)
             {
-                foreach (var container in instance.Containers.Containers)
+                foreach (var container in instance.Pod.Containers)
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
                     log.Log(
-                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
+                        $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
@@ -92,7 +92,7 @@ namespace ContinuousTests
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
             var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {
@@ -123,10 +123,10 @@ namespace ContinuousTests
             try
             {
                 var info = n.GetDebugInfo();
-                if (info == null || string.IsNullOrEmpty(info.id)) return false;
+                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
 
-                log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
-                LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName()));
+                log.Log($"Codex version: '{info.Version.Version}' revision: '{info.Version.Revision}'");
+                LogReplacements.Add(new BaseLogStringReplacement(info.Id, n.GetName()));
             }
             catch
             {

Tests/CodexContinuousTests/Tests/PeersTest.cs

@@ -24,12 +24,12 @@ namespace CodexContinuousTests.Tests
             var allInfos = Nodes.Select(n =>
             {
                 var info = n.GetDebugInfo();
-                Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}");
-                Log.AddStringReplace(info.table.localNode.nodeId, n.GetName());
+                Log.Log($"{n.GetName()} = {info.Table.LocalNode.NodeId}");
+                Log.AddStringReplace(info.Table.LocalNode.NodeId, n.GetName());
                 return info;
             }).ToArray();
 
-            var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray();
+            var allIds = allInfos.Select(i => i.Table.LocalNode.NodeId).ToArray();
             var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
 
             if (errors.Any())
@@ -41,13 +41,13 @@ namespace CodexContinuousTests.Tests
         private string AreAllPresent(ICodexNode n, string[] allIds)
         {
             var info = n.GetDebugInfo();
-            var known = info.table.nodes.Select(n => n.nodeId).ToArray();
-            var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
+            var known = info.Table.Nodes.Select(n => n.NodeId).ToArray();
+            var expected = allIds.Where(i => i != info.Table.LocalNode.NodeId).ToArray();
 
             if (!expected.All(ex => known.Contains(ex)))
             {
                 var nl = Environment.NewLine;
-                return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" +
+                return $"{nl}At node '{info.Table.LocalNode.NodeId}'{nl}" +
                     $"Not all of{nl}'{string.Join(",", expected)}'{nl}" +
                     $"were present in routing table:{nl}'{string.Join(",", known)}'";
             }

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -132,7 +132,7 @@ namespace CodexTests.BasicTests
 
         private const string BytesStoredMetric = "codexRepostoreBytesUsed";
 
-        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc)
+        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc)
         {
             ScopedTestFiles(() =>
             {
@@ -154,7 +154,7 @@ namespace CodexTests.BasicTests
                     var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
 
                     return high > newBytes && newBytes > low;
-                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute));
 
                 FileUtils.TrackedFile? downloadedFile = null;
                 LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId));

Tests/CodexTests/BasicTests/DiscordBotTests.cs

@@ -0,0 +1,110 @@
+using CodexContractsPlugin;
+using CodexDiscordBotPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class DiscordBotTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void BotRewardTest()
+        {
+            var myAccount = EthAccount.GenerateNew();
+
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 100000.TestTokens();
+            var fileSize = 11.MB();
+
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var contracts = Ci.StartCodexContracts(geth);
+
+            // start bot and rewarder
+            var gethInfo = new DiscordBotGethInfo(
+                host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host,
+                port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port,
+                privKey: geth.StartResult.Account.PrivateKey,
+                marketplaceAddress: contracts.Deployment.MarketplaceAddress,
+                tokenAddress: contracts.Deployment.TokenAddress,
+                abi: contracts.Deployment.Abi
+            );
+            var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
+                name: "bot",
+                token: "aaa",
+                serverName: "ThatBen's server",
+                adminRoleName: "bottest-admins",
+                adminChannelName: "admin-channel",
+                rewardChannelName: "rewards-channel",
+                kubeNamespace: "notneeded",
+                gethInfo: gethInfo
+            ));
+            var botContainer = bot.Containers.Single();
+            Ci.DeployRewarderBot(new RewarderBotStartupConfig(
+                //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host,
+                //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port,
+                discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host,
+                discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port,
+                intervalMinutes: "1",
+                historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3),
+                gethInfo: gethInfo,
+                dataPath: null
+            ));
+
+            var numberOfHosts = 3;
+
+            for (var i = 0; i < numberOfHosts; i++)
+            {
+                var seller = AddCodex(s => s
+                    .WithName("Seller")
+                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                    {
+                        ContractClock = CodexLogLevel.Trace,
+                    })
+                    .WithStorageQuota(11.GB())
+                    .EnableMarketplace(geth, contracts, m => m
+                        .WithAccount(myAccount)
+                        .WithInitial(10.Eth(), sellerInitialBalance)
+                        .AsStorageNode()
+                        .AsValidator()));
+
+                var availability = new StorageAvailability(
+                    totalSpace: 10.GB(),
+                    maxDuration: TimeSpan.FromMinutes(30),
+                    minPriceForTotalSpace: 1.TestTokens(),
+                    maxCollateral: 20.TestTokens()
+                );
+                seller.Marketplace.MakeStorageAvailable(availability);
+            }
+
+            var testFile = GenerateTestFile(fileSize);
+
+            var buyer = AddCodex(s => s
+                .WithName("Buyer")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithAccount(myAccount)
+                    .WithInitial(10.Eth(), buyerInitialBalance)));
+
+            var contentId = buyer.UploadFile(testFile);
+
+            var purchase = new StoragePurchaseRequest(contentId)
+            {
+                PricePerSlotPerSecond = 2.TestTokens(),
+                RequiredCollateral = 10.TestTokens(),
+                MinRequiredNumberOfNodes = 5,
+                NodeFailureTolerance = 2,
+                ProofProbability = 5,
+                Duration = TimeSpan.FromMinutes(6),
+                Expiry = TimeSpan.FromMinutes(5)
+            };
+
+            var purchaseContract = buyer.Marketplace.RequestStorage(purchase);
+
+            purchaseContract.WaitForStorageContractStarted();
+
+            purchaseContract.WaitForStorageContractFinished();
+        }
+    }
+}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -1,12 +1,9 @@
-using CodexContractsPlugin;
-using CodexPlugin;
+using CodexPlugin;
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
-using Nethereum.Hex.HexConvertors.Extensions;
 using NUnit.Framework;
 using Utils;
-using Request = CodexContractsPlugin.Marketplace.Request;
 
 namespace CodexTests.BasicTests
 {
@@ -50,76 +47,6 @@ namespace CodexTests.BasicTests
             metrics[1].AssertThat("libp2p_peers", Is.EqualTo(1));
         }
 
-        [Test]
-        public void MarketplaceExample()
-        {
-            var sellerInitialBalance = 234.TestTokens();
-            var buyerInitialBalance = 100000.TestTokens();
-            var fileSize = 10.MB();
-
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
-
-            var seller = AddCodex(s => s
-                .WithName("Seller")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn))
-                .WithStorageQuota(11.GB())
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), sellerInitialBalance)
-                    .AsStorageNode()
-                    .AsValidator()));
-
-            AssertBalance(contracts, seller, Is.EqualTo(sellerInitialBalance));
-
-            var availability = new StorageAvailability(
-                totalSpace: 10.GB(),
-                maxDuration: TimeSpan.FromMinutes(30),
-                minPriceForTotalSpace: 1.TestTokens(),
-                maxCollateral: 20.TestTokens()
-            );
-            seller.Marketplace.MakeStorageAvailable(availability);
-
-            var testFile = GenerateTestFile(fileSize);
-
-            var buyer = AddCodex(s => s
-                .WithName("Buyer")
-                .WithBootstrapNode(seller)
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), buyerInitialBalance)));
-
-            AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance));
-
-            var contentId = buyer.UploadFile(testFile);
-
-            var purchase = new StoragePurchaseRequest(contentId)
-            {
-                PricePerSlotPerSecond = 2.TestTokens(),
-                RequiredCollateral = 10.TestTokens(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
-            };
-
-            var purchaseContract = buyer.Marketplace.RequestStorage(purchase);
-
-            purchaseContract.WaitForStorageContractStarted();
-
-            AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
-
-            var request = GetOnChainStorageRequest(contracts);
-            AssertStorageRequest(request, purchase, contracts, buyer);
-            AssertSlotFilledEvents(contracts, purchase, request, seller);
-            AssertContractSlot(contracts, request, 0, seller);
-
-            purchaseContract.WaitForStorageContractFinished();
-
-            AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
-            AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
-            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
-        }
-
         [Test]
         public void GethBootstrapTest()
         {
@@ -136,43 +63,5 @@ namespace CodexTests.BasicTests
             Assert.That(bootN, Is.EqualTo(followN));
             Assert.That(discN, Is.LessThan(bootN));
         }
-
-        private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller)
-        {
-            // Expect 1 fulfilled event for the purchase.
-            var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange());
-            Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1));
-            CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId);
-
-            // Expect 1 filled-slot event for each slot in the purchase.
-            var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange());
-            Assert.That(filledSlotEvents.Length, Is.EqualTo(purchase.MinRequiredNumberOfNodes));
-            for (var i = 0; i < purchase.MinRequiredNumberOfNodes; i++)
-            {
-                var filledSlotEvent = filledSlotEvents.Single(e => e.SlotIndex == i);
-                Assert.That(filledSlotEvent.RequestId.ToHex(), Is.EqualTo(request.RequestId.ToHex()));
-                Assert.That(filledSlotEvent.Host, Is.EqualTo(seller.EthAddress));
-            }
-        }
-
-        private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)
-        {
-            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started));
-            Assert.That(request.ClientAddress, Is.EqualTo(buyer.EthAddress));
-            Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes));
-        }
-
-        private Request GetOnChainStorageRequest(ICodexContracts contracts)
-        {
-            var requests = contracts.GetStorageRequests(GetTestRunTimeRange());
-            Assert.That(requests.Length, Is.EqualTo(1));
-            return requests.Single();
-        }
-
-        private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex, ICodexNode expectedSeller)
-        {
-            var slotHost = contracts.GetSlotHost(request, contractSlotIndex);
-            Assert.That(slotHost, Is.EqualTo(expectedSeller.EthAddress));
-        }
     }
 }

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -0,0 +1,121 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class MarketplaceTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void MarketplaceExample()
+        {
+            var hostInitialBalance = 234.TestTokens();
+            var clientInitialBalance = 100000.TestTokens();
+            var fileSize = 10.MB();
+
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var contracts = Ci.StartCodexContracts(geth);
+            
+            var numberOfHosts = 5;
+            var hosts = AddCodex(numberOfHosts, s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                {
+                    ContractClock = CodexLogLevel.Trace,
+                })
+                .WithStorageQuota(11.GB())
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
+
+            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
+            foreach (var host in hosts)
+            {
+                AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
+
+                var availability = new StorageAvailability(
+                    totalSpace: 10.GB(),
+                    maxDuration: TimeSpan.FromMinutes(30),
+                    minPriceForTotalSpace: 1.TestTokens(),
+                    maxCollateral: 20.TestTokens()
+                );
+                host.Marketplace.MakeStorageAvailable(availability);
+            }
+
+            var testFile = GenerateTestFile(fileSize);
+
+            var client = AddCodex(s => s
+                .WithName("Client")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), clientInitialBalance)));
+
+            AssertBalance(contracts, client, Is.EqualTo(clientInitialBalance));
+
+            var contentId = client.UploadFile(testFile);
+
+            var purchase = new StoragePurchaseRequest(contentId)
+            {
+                PricePerSlotPerSecond = 2.TestTokens(),
+                RequiredCollateral = 10.TestTokens(),
+                MinRequiredNumberOfNodes = 5,
+                NodeFailureTolerance = 2,
+                ProofProbability = 5,
+                Duration = TimeSpan.FromMinutes(5),
+                Expiry = TimeSpan.FromMinutes(4)
+            };
+
+            var purchaseContract = client.Marketplace.RequestStorage(purchase);
+
+            WaitForAllSlotFilledEvents(contracts, purchase, geth);
+
+            purchaseContract.WaitForStorageContractStarted();
+
+            var request = GetOnChainStorageRequest(contracts, geth);
+            AssertStorageRequest(request, purchase, contracts, client);
+            AssertContractSlot(contracts, request, 0);
+
+            purchaseContract.WaitForStorageContractFinished();
+
+            AssertBalance(contracts, client, Is.LessThan(clientInitialBalance), "Buyer was not charged for storage.");
+            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
+        }
+
+        private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)
+        {
+            Time.Retry(() =>
+            {
+                var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange());
+                var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
+
+                Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
+
+                if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
+            }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
+        }
+
+        private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)
+        {
+            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started));
+            Assert.That(request.ClientAddress, Is.EqualTo(buyer.EthAddress));
+            Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes));
+        }
+
+        private Request GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth)
+        {
+            var requests = contracts.GetStorageRequests(geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()));
+            Assert.That(requests.Length, Is.EqualTo(1));
+            return requests.Single();
+        }
+
+        private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex)
+        {
+            var slotHost = contracts.GetSlotHost(request, contractSlotIndex);
+            Assert.That(slotHost?.Address, Is.Not.Null);
+        }
+    }
+}

Tests/CodexTests/BasicTests/NetworkIsolationTest.cs

@@ -19,7 +19,7 @@ namespace CodexTests.BasicTests
         {
             node = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
+            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait));
         }
 
         [Test]
@@ -27,7 +27,7 @@ namespace CodexTests.BasicTests
         {
             var myNode = Ci.StartCodexNode();
 
-            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
+            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects));
 
             try
             {

Tests/CodexTests/CodexTests.csproj

@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\MetricsPlugin\MetricsPlugin.csproj" />

Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs

@@ -0,0 +1,83 @@
+using DistTestCore;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class ClusterDiscSpeedTests : DistTest
+    {
+        private readonly Random random = new Random();
+
+        [Test]
+        [Combinatorial]
+        public void DiscSpeedTest(
+            [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb
+        )
+        {
+            long targetSize = (long)(1024 * 1024 * 1024) * 2;
+            long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
+
+            var filename = nameof(DiscSpeedTest);
+
+            Thread.Sleep(2000);
+            if (File.Exists(filename)) File.Delete(filename);
+            Thread.Sleep(2000);
+            var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
+            Thread.Sleep(2000);
+            var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
+            
+            Log($"Write speed: {writeSpeed} per second.");
+            Log($"Read speed: {readSpeed} per second.");
+        }
+
+        private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename)
+        {
+            long bytesWritten = 0;
+            var buffer = new byte[bufferSizeBytes];
+            random.NextBytes(buffer);
+
+            var sw = Stopwatch.Begin(GetTestLog());
+            using (var stream = File.OpenWrite(filename))
+            {
+                while (bytesWritten < targetSize)
+                {
+                    long remaining = targetSize - bytesWritten;
+                    long toWrite = Math.Min(bufferSizeBytes, remaining);
+
+                    stream.Write(buffer, 0, Convert.ToInt32(toWrite));
+                    bytesWritten += toWrite;
+                }
+            }
+            var duration = sw.End("WriteTime");
+            double totalSeconds = duration.TotalSeconds;
+            double totalBytes = bytesWritten;
+            double bytesPerSecond = totalBytes / totalSeconds;
+            return new ByteSize(Convert.ToInt64(bytesPerSecond));
+        }
+
+        private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename)
+        {
+            long bytesRead = 0;
+            var buffer = new byte[bufferSizeBytes];
+            var sw = Stopwatch.Begin(GetTestLog());
+            using (var stream = File.OpenRead(filename))
+            {
+                while (bytesRead < targetSize)
+                {
+                    long remaining = targetSize - bytesRead;
+                    long toRead = Math.Min(bufferSizeBytes, remaining);
+
+                    var r = stream.Read(buffer, 0, Convert.ToInt32(toRead));
+                    bytesRead += r;
+                }
+            }
+            var duration = sw.End("ReadTime");
+            double totalSeconds = duration.TotalSeconds;
+            double totalBytes = bytesRead;
+            double bytesPerSecond = totalBytes / totalSeconds;
+            return new ByteSize(Convert.ToInt64(bytesPerSecond));
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -0,0 +1,120 @@
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class MultiPeerDownloadTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [DontDownloadLogs]
+        [UseLongTimeouts]
+        [Combinatorial]
+        public void MultiPeerDownload(
+            [Values(5, 10, 20)] int numberOfHosts,
+            [Values(100, 1000)] int fileSize
+        )
+        {
+            var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = hosts[0].UploadFile(file);
+            var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6);
+
+            var uploadLog = Ci.DownloadLog(hosts[0]);
+            var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
+            var blockCids = uploadLog
+                .FindLinesThatContain("Putting block into network store")
+                .Select(s =>
+                {
+                    var start = s.IndexOf("cid=") + 4;
+                    var end = s.IndexOf(" count=");
+                    var len = end - start;
+                    return s.Substring(start, len);
+                })
+                .ToArray();
+
+            Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks));
+
+            foreach (var h in hosts) h.DownloadContent(cid);
+
+            var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace));
+            var resultFile = client.DownloadContent(cid);
+            resultFile!.AssertIsEqual(file);
+
+            var downloadLog = Ci.DownloadLog(client);
+            var host = string.Empty;
+            var blockCidHostMap = new Dictionary<string, string>();
+            downloadLog.IterateLines(line =>
+            {
+                if (line.Contains("peer=") && line.Contains(" len="))
+                {
+                    var start = line.IndexOf("peer=") + 5;
+                    var end = line.IndexOf(" len=");
+                    var len = end - start;
+                    host = line.Substring(start, len);
+                }
+                else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key"))
+                {
+                    var start = line.IndexOf("cid=") + 4;
+                    var end = line.IndexOf(" count=");
+                    var len = end - start;
+                    var blockCid = line.Substring(start, len);
+
+                    blockCidHostMap.Add(blockCid, host);
+                    host = string.Empty;
+                }
+            });
+
+            var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value));
+            //PrintFullMap(blockCidHostMap);
+            PrintOverview(blockCidHostMap);
+
+            Log("Expected number of blocks: " + expectedNumberOfBlocks);
+            Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length);
+            Log("Total blocks fetched by hosts: " + totalFetched);
+            Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks));
+        }
+
+        private void PrintOverview(Dictionary<string, string> blockCidHostMap)
+        {
+            var overview = new Dictionary<string, int>();
+            foreach (var pair in blockCidHostMap)
+            {
+                if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1);
+                else overview[pair.Value]++;
+            }
+
+            Log("Blocks fetched per host:");
+            foreach (var pair in overview)
+            {
+                Log($"Host: {pair.Key} = {pair.Value}");
+            }
+        }
+
+        private void PrintFullMap(Dictionary<string, string> blockCidHostMap)
+        {
+            Log("Per block, host it was fetched from:");
+            foreach (var pair in blockCidHostMap)
+            {
+                if (string.IsNullOrEmpty(pair.Value))
+                {
+                    Log($"block: {pair.Key} = Not seen");
+                }
+                else
+                {
+                    Log($"block: {pair.Key} = '{pair.Value}'");
+                }
+            }
+        }
+
+        private long RoundUp(long filesize, long blockSize)
+        {
+            double f = filesize;
+            double b = blockSize;
+
+            var result = Math.Ceiling(f / b);
+            return Convert.ToInt64(result);
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs

@@ -0,0 +1,39 @@
+using CodexPlugin;
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests
+{
+    [TestFixture]
+    public class OneClientLargeFileTests : CodexDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [UseLongTimeouts]
+        public void OneClientLargeFile([Values(
+            256,
+            512,
+            1024, // GB
+            2048,
+            4096,
+            8192,
+            16384,
+            32768,
+            65536,
+            131072
+        )] int sizeMb)
+        {
+            var testFile = GenerateTestFile(sizeMb.MB());
+
+            var node = AddCodex(s => s
+                .WithLogLevel(CodexLogLevel.Warn)
+                .WithStorageQuota((sizeMb + 10).MB())
+            );
+            var contentId = node.UploadFile(testFile);
+            var downloadedFile = node.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs

@@ -0,0 +1,129 @@
+using CodexPlugin;
+using DistTestCore;
+using FileUtils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.ScalabilityTests;
+
+[TestFixture]
+public class ScalabilityTests : CodexDistTest
+{
+    private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests";
+    private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests";
+
+    /// <summary>
+    /// We upload a file to node A, then download it with B.
+    /// Then we stop node A, and download again with node C.
+    /// </summary>
+    [Test]
+    [Combinatorial]
+    [UseLongTimeouts]
+    [DontDownloadLogs]
+    public void ShouldMaintainFileInNetwork(
+        [Values(10, 40, 80, 100)] int numberOfNodes,
+        [Values(100, 1000, 5000, 10000)] int fileSizeInMb,
+        [Values(true, false)] bool usePatchedImage
+    )
+    {
+        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
+
+        var logLevel = CodexLogLevel.Info;
+
+        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var nodes = AddCodex(numberOfNodes - 1, s => s
+            .WithBootstrapNode(bootstrap)
+            .WithLogLevel(logLevel)
+            .WithStorageQuota((fileSizeInMb + 50).MB())
+        ).ToList();
+
+        var uploader = nodes.PickOneRandom();
+        var downloader = nodes.PickOneRandom();
+
+        var testFile = GenerateTestFile(fileSizeInMb.MB());
+        var contentId = uploader.UploadFile(testFile);
+        var downloadedFile = downloader.DownloadContent(contentId);
+
+        downloadedFile!.AssertIsEqual(testFile);
+
+        uploader.Stop(true);
+
+        var otherDownloader = nodes.PickOneRandom();
+        downloadedFile = otherDownloader.DownloadContent(contentId);
+
+        downloadedFile!.AssertIsEqual(testFile);
+    }
+
+    /// <summary>
+    /// We upload a file to each node, to put a more wide-spread load on the network.
+    /// Then we run the same test as ShouldMaintainFileInNetwork.
+    /// </summary>
+    [Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")]
+    [Test]
+    [Combinatorial]
+    [UseLongTimeouts]
+    [DontDownloadLogs]
+    public void EveryoneGetsAFile(
+        [Values(10, 40, 80, 100)] int numberOfNodes,
+        [Values(100, 1000)] int fileSizeInMb,
+        [Values(true, false)] bool usePatchedImage
+    )
+    {
+        CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage;
+
+        var logLevel = CodexLogLevel.Info;
+
+        var bootstrap = AddCodex(s => s.WithLogLevel(logLevel));
+        var nodes = AddCodex(numberOfNodes - 1, s => s
+            .WithBootstrapNode(bootstrap)
+            .WithLogLevel(logLevel)
+            .WithStorageQuota((fileSizeInMb + 50).MB())
+        ).ToList();
+
+        var pairTasks = nodes.Select(n =>
+        {
+            return Task.Run(() =>
+            {
+                var file = GenerateTestFile(fileSizeInMb.MB());
+                var cid = n.UploadFile(file);
+                return new NodeFilePair(n, file, cid);
+            });
+        });
+
+        var pairs = pairTasks.Select(t => Time.Wait(t)).ToList();
+
+        RunDoubleDownloadTest(
+            pairs.PickOneRandom(),
+            pairs.PickOneRandom(),
+            pairs.PickOneRandom()
+        );
+    }
+
+    private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2)
+    {
+        var expectedFile = source.File;
+        var cid = source.Cid;
+
+        var file1 = dl1.Node.DownloadContent(cid);
+        file1!.AssertIsEqual(expectedFile);
+
+        source.Node.Stop(true);
+
+        var file2 = dl2.Node.DownloadContent(cid);
+        file2!.AssertIsEqual(expectedFile);
+    }
+
+    public class NodeFilePair
+    {
+        public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid)
+        {
+            Node = node;
+            File = file;
+            Cid = cid;
+        }
+
+        public ICodexNode Node { get; }
+        public TrackedFile File { get; }
+        public ContentId Cid { get; }
+    }
+}

Tests/DistTestCore/Configuration.cs

@@ -24,6 +24,9 @@ namespace DistTestCore
             this.dataFilesPath = dataFilesPath;
         }
 
+        /// <summary>
+        /// Does not override [DontDownloadLogs] attribute.
+        /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
@@ -36,7 +39,7 @@ namespace DistTestCore
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay(),
+                retryDelay: timeSet.K8sOperationRetryDelay(),
                 kubernetesNamespace: k8sNamespace
             );
 

Tests/DistTestCore/DistTest.cs

@@ -98,7 +98,7 @@ namespace DistTestCore
             }
             catch (Exception ex)
             {
-                fixtureLog.Error("Cleanup failed: " + ex.Message);
+                fixtureLog.Error("Cleanup failed: " + ex);
                 GlobalTestFailure.HasFailed = true;
             }
         }
@@ -236,9 +236,19 @@ namespace DistTestCore
         }
 
         private bool ShouldUseLongTimeouts()
+        {
+            return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();
+        }
+
+        private bool HasDontDownloadAttribute()
+        {
+            return CurrentTestMethodHasAttribute<DontDownloadLogsAttribute>();
+        }
+
+        private bool CurrentTestMethodHasAttribute<T>() where T : PropertyAttribute
         {
             // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
+            // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out
             // if the attribute is present.
             var currentTest = TestContext.CurrentContext.Test;
             var className = currentTest.ClassName;
@@ -247,7 +257,7 @@ namespace DistTestCore
             var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
             var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
 
-            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
+            return testMethods.Any(m => m.GetCustomAttribute<T>() != null);
         }
 
         private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
@@ -268,9 +278,10 @@ namespace DistTestCore
         private bool ShouldDownloadAllLogs(TestStatus testStatus)
         {
             if (configuration.AlwaysDownloadContainerLogs) return true;
+            if (!IsDownloadingLogsEnabled()) return false;
             if (testStatus == TestStatus.Failed)
             {
-                return IsDownloadingLogsEnabled();
+                return true;
             }
 
             return false;
@@ -288,8 +299,7 @@ namespace DistTestCore
 
         private bool IsDownloadingLogsEnabled()
         {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
+            return !HasDontDownloadAttribute();
         }
     }
 

Tests/DistTestCore/DontDownloadLogsAttribute.cs

@@ -3,11 +3,11 @@
 namespace DistTestCore
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
+    public class DontDownloadLogsAttribute : PropertyAttribute
     {
         public const string DontDownloadKey = "DontDownloadLogs";
 
-        public DontDownloadLogsOnFailureAttribute()
+        public DontDownloadLogsAttribute()
             : base(DontDownloadKey)
         {
         }

Tests/DistTestCore/Helpers/AssertHelpers.cs

@@ -14,7 +14,7 @@ namespace DistTestCore.Helpers
                 Time.WaitUntil(() => {
                     var c = constraint.Resolve();
                     return c.ApplyTo(actual()).IsSuccess;
-                });
+                }, "RetryAssert: " + message);
             }
             catch (TimeoutException)
             {

Tests/DistTestCore/TestLifecycle.cs

@@ -13,7 +13,7 @@ namespace DistTestCore
         private const string TestsType = "dist-tests";
         private readonly EntryPoint entryPoint;
         private readonly Dictionary<string, string> metadata; 
-        private readonly List<RunningContainers> runningContainers = new();
+        private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
@@ -65,12 +65,12 @@ namespace DistTestCore
             return DateTime.UtcNow - TestStart;
         }
 
-        public void OnContainersStarted(RunningContainers rc)
+        public void OnContainersStarted(RunningPod rc)
         {
             runningContainers.Add(rc);
         }
 
-        public void OnContainersStopped(RunningContainers rc)
+        public void OnContainersStopped(RunningPod rc)
         {
             runningContainers.Remove(rc);
         }
@@ -93,13 +93,20 @@ namespace DistTestCore
 
         public void DownloadAllLogs()
         {
-            foreach (var rc in runningContainers)
+            try
             {
-                foreach (var c in rc.Containers)
+                foreach (var rc in runningContainers)
                 {
-                    CoreInterface.DownloadLog(c);
+                    foreach (var c in rc.Containers)
+                    {
+                        CoreInterface.DownloadLog(c);
+                    }
                 }
             }
+            catch (Exception ex)
+            {
+                Log.Error("Exception during log download: " + ex);
+            }
         }
     }
 }

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -117,6 +117,41 @@ namespace FrameworkTests.NethereumWorkflow
 
             Assert.That(notFound, Is.Null);
         }
+
+        [Test]
+        public void FailsToFindBlockBeforeFrontOfChain_history()
+        {
+            var first = blocks.First().Value;
+
+            var notFound = finder.GetHighestBlockNumberBefore(first.JustBefore);
+
+            Assert.That(notFound, Is.Null);
+        }
+
+        [Test]
+        public void FailsToFindBlockAfterTailOfChain_future()
+        {
+            var last = blocks.Last().Value;
+
+            var notFound = finder.GetLowestBlockNumberAfter(last.JustAfter);
+
+            Assert.That(notFound, Is.Null);
+        }
+
+        [Test]
+        public void RunThrough()
+        {
+            foreach (var pair in blocks)
+            {
+                finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
+                finder.GetHighestBlockNumberBefore(pair.Value.Time);
+                finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
+
+                finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
+                finder.GetLowestBlockNumberAfter(pair.Value.Time);
+                finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
+            }
+        }
     }
 
     public class Block
@@ -131,5 +166,10 @@ namespace FrameworkTests.NethereumWorkflow
         public DateTime Time { get; }
         public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
         public DateTime JustAfter { get { return Time.AddSeconds(1); } }
+
+        public override string ToString()
+        {
+            return $"[{Number}]";
+        }
     }
 }

Tools/BiblioTech/BiblioTech.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -12,7 +12,6 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
-    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CommandHandler.cs

@@ -26,12 +26,10 @@ namespace BiblioTech
             Program.AdminChecker.SetGuild(guild);
             Program.Log.Log($"Initializing for guild: '{guild.Name}'");
 
-            var roleController = new RoleController(client);
-            var rewardsApi = new RewardsApi(roleController);
-
             var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray();
             if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel");
             Program.AdminChecker.SetAdminChannel(adminChannels.First());
+            Program.RoleDriver = new RoleDriver(client);
 
             var builders = commands.Select(c =>
             {
@@ -62,8 +60,6 @@ namespace BiblioTech
                 var json = JsonConvert.SerializeObject(exception.Errors, Formatting.Indented);
                 Program.Log.Error(json);
             }
-
-            rewardsApi.Start();
         }
 
         private async Task SlashCommandHandler(SocketSlashCommand command)

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -31,8 +31,14 @@ namespace BiblioTech.Commands
                 return;
             }
 
-            var eth = gethNode.GetEthBalance(addr);
-            var testTokens = contracts.GetTestTokenBalance(addr);
+            var eth = 0.Eth();
+            var testTokens = 0.TestTokens();
+
+            await Task.Run(() =>
+            {
+                eth = gethNode.GetEthBalance(addr);
+                testTokens = contracts.GetTestTokenBalance(addr);
+            });
 
             await context.Followup($"{context.Command.User.Username} has {eth} and {testTokens}.");
         }

Tools/BiblioTech/Commands/MarketCommand.cs

@@ -0,0 +1,58 @@
+using BiblioTech.Options;
+using DiscordRewards;
+using System.Globalization;
+using Utils;
+
+namespace BiblioTech.Commands
+{
+    public class MarketCommand : BaseCommand
+    {
+        public override string Name => "market";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Fetch some insights about current market conditions.";
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            await context.Followup(GetInsights());
+        }
+
+        private string[] GetInsights()
+        {
+            var result = Program.Averages.SelectMany(GetInsight).ToArray();
+            if (result.Length > 0)
+            {
+                result = new[]
+                {
+                    "No market insights available."
+                };
+            }
+            return result;
+        }
+
+        private string[] GetInsight(MarketAverage avg)
+        {
+            var timeRange = TimeSpan.FromSeconds(avg.TimeRangeSeconds);
+            var headerLine = $"[Last {Time.FormatDuration(timeRange)}] ({avg.NumberOfFinished} Contracts finished)";
+
+            if (avg.NumberOfFinished == 0)
+            {
+                return new[] { headerLine }; 
+            }
+
+            return new[]
+            {
+                headerLine,
+                $"Price: {Format(avg.Price)}",
+                $"Size: {Format(avg.Size)}",
+                $"Duration: {Format(avg.Duration)}",
+                $"Collateral: {Format(avg.Collateral)}",
+                $"ProofProbability: {Format(avg.ProofProbability)}"
+            };
+        }
+
+        private string Format(float f)
+        {
+            return f.ToString("F3", CultureInfo.InvariantCulture);
+        }
+    }
+}

Tools/BiblioTech/Commands/MintCommand.cs

@@ -6,8 +6,6 @@ namespace BiblioTech.Commands
 {
     public class MintCommand : BaseGethCommand
     {
-        private readonly Ether defaultEthToSend = 10.Eth();
-        private readonly TestToken defaultTestTokensToMint = 1024.TestTokens();
         private readonly UserOption optionalUser = new UserOption(
             description: "If set, mint tokens for this user. (Optional, admin-only)",
             isRequired: false);
@@ -35,8 +33,14 @@ namespace BiblioTech.Commands
 
             var report = new List<string>();
 
-            var sentEth = ProcessEth(gethNode, addr, report);
-            var mintedTokens = ProcessTokens(contracts, addr, report);
+            Transaction<Ether>? sentEth = null;
+            Transaction<TestToken>? mintedTokens = null;
+
+            await Task.Run(() =>
+            {
+                sentEth = ProcessEth(gethNode, addr, report);
+                mintedTokens = ProcessTokens(contracts, addr, report);
+            });
 
             Program.UserRepo.AddMintEventForUser(userId, addr, sentEth, mintedTokens);
 
@@ -47,9 +51,10 @@ namespace BiblioTech.Commands
         {
             if (ShouldMintTestTokens(contracts, addr))
             {
-                var transaction = contracts.MintTestTokens(addr, defaultTestTokensToMint);
-                report.Add($"Minted {defaultTestTokensToMint} {FormatTransactionLink(transaction)}");
-                return new Transaction<TestToken>(defaultTestTokensToMint, transaction);
+                var tokens = Program.Config.MintTT.TestTokens();
+                var transaction = contracts.MintTestTokens(addr, tokens);
+                report.Add($"Minted {tokens} {FormatTransactionLink(transaction)}");
+                return new Transaction<TestToken>(tokens, transaction);
             }
             
             report.Add("TestToken balance over threshold. (No TestTokens minted.)");
@@ -60,9 +65,10 @@ namespace BiblioTech.Commands
         {
             if (ShouldSendEth(gethNode, addr))
             {
-                var transaction = gethNode.SendEth(addr, defaultEthToSend);
-                report.Add($"Sent {defaultEthToSend} {FormatTransactionLink(transaction)}");
-                return new Transaction<Ether>(defaultEthToSend, transaction);
+                var eth = Program.Config.SendEth.Eth();
+                var transaction = gethNode.SendEth(addr, eth);
+                report.Add($"Sent {eth} {FormatTransactionLink(transaction)}");
+                return new Transaction<Ether>(eth, transaction);
             }
             report.Add("Eth balance is over threshold. (No Eth sent.)");
             return null;
@@ -71,13 +77,13 @@ namespace BiblioTech.Commands
         private bool ShouldMintTestTokens(ICodexContracts contracts, EthAddress addr)
         {
             var testTokens = contracts.GetTestTokenBalance(addr);
-            return testTokens.Amount < 64m;
+            return testTokens.Amount < Program.Config.MintTT;
         }
 
         private bool ShouldSendEth(IGethNode gethNode, EthAddress addr)
         {
             var eth = gethNode.GetEthBalance(addr);
-            return eth.Eth < 1.0m;
+            return eth.Eth < Program.Config.SendEth;
         }
 
         private string FormatTransactionLink(string transaction)

Tools/BiblioTech/Configuration.cs

@@ -19,9 +19,20 @@ namespace BiblioTech
         [Uniform("admin-channel-name", "ac", "ADMINCHANNELNAME", true, "Name of the Discord server channel where admin commands are allowed.")]
         public string AdminChannelName { get; set; } = "admin-channel";
 
-        [Uniform("rewards-channel-name", "ac", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")]
+        [Uniform("rewards-channel-name", "rc", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")]
         public string RewardsChannelName { get; set; } = "";
 
+        [Uniform("chain-events-channel-name", "cc", "CHAINEVENTSCHANNELNAME", false, "Name of the Discord server channel where chain events will be posted.")]
+        public string ChainEventsChannelName { get; set; } = "";
+
+        [Uniform("reward-api-port", "rp", "REWARDAPIPORT", false, "TCP listen port for the reward API.")]
+        public int RewardApiPort { get; set; } = 31080;
+
+        [Uniform("send-eth", "se", "SENDETH", false, "Amount of Eth send by the mint command. Default: 10.")]
+        public int SendEth { get; set; } = 10;
+
+        [Uniform("mint-tt", "mt", "MINTTT", false, "Amount of TestTokens minted by the mint command. Default: 1073741824")]
+        public int MintTT { get; set; } = 1073741824;
 
         public string EndpointsPath
         {

Tools/BiblioTech/Program.cs

@@ -1,7 +1,9 @@
 using ArgsUniform;
 using BiblioTech.Commands;
+using BiblioTech.Rewards;
 using Discord;
 using Discord.WebSocket;
+using DiscordRewards;
 using Logging;
 
 namespace BiblioTech
@@ -13,7 +15,9 @@ namespace BiblioTech
         public static Configuration Config { get; private set; } = null!;
         public static UserRepo UserRepo { get; } = new UserRepo();
         public static AdminChecker AdminChecker { get; private set; } = null!;
+        public static IDiscordRoleDriver RoleDriver { get; set; } = null!;
         public static ILog Log { get; private set; } = null!;
+        public static MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
 
         public static Task Main(string[] args)
         {
@@ -29,10 +33,10 @@ namespace BiblioTech
             EnsurePath(Config.UserDataPath);
             EnsurePath(Config.EndpointsPath);
 
-            return new Program().MainAsync();
+            return new Program().MainAsync(args);
         }
 
-        public async Task MainAsync()
+        public async Task MainAsync(string[] args)
         {
             Log.Log("Starting Codex Discord Bot...");
             client = new DiscordSocketClient();
@@ -47,15 +51,25 @@ namespace BiblioTech
                 sprCommand,
                 associateCommand,
                 notifyCommand,
-                new AdminCommand(sprCommand)
+                new AdminCommand(sprCommand),
+                new MarketCommand()
             );
 
             await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
             await client.StartAsync();
-
             AdminChecker = new AdminChecker();
 
+            var builder = WebApplication.CreateBuilder(args);
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Config.RewardApiPort);
+            });
+            builder.Services.AddControllers();
+            var app = builder.Build();
+            app.MapControllers();
+
             Log.Log("Running...");
+            await app.RunAsync();
             await Task.Delay(-1);
         }
 

Tools/BiblioTech/Properties/launchSettings.json

@@ -0,0 +1,12 @@
+{
+  "profiles": {
+    "BiblioTech": {
+      "commandName": "Project",
+      "launchBrowser": true,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "https://localhost:52960;http://localhost:52961"
+    }
+  }
+}

Tools/BiblioTech/Rewards/RewardContext.cs

@@ -0,0 +1,102 @@
+using Discord.WebSocket;
+using Discord;
+using DiscordRewards;
+
+namespace BiblioTech.Rewards
+{
+    public class RewardContext
+    {
+        private readonly Dictionary<ulong, IGuildUser> users;
+        private readonly Dictionary<ulong, RoleReward> roles;
+        private readonly SocketTextChannel? rewardsChannel;
+
+        public RewardContext(Dictionary<ulong, IGuildUser> users, Dictionary<ulong, RoleReward> roles, SocketTextChannel? rewardsChannel)
+        {
+            this.users = users;
+            this.roles = roles;
+            this.rewardsChannel = rewardsChannel;
+        }
+
+        public async Task ProcessGiveRewardsCommand(UserReward[] rewards)
+        {
+            foreach (var rewardCommand in rewards)
+            {
+                if (roles.ContainsKey(rewardCommand.RewardCommand.RewardId))
+                {
+                    var role = roles[rewardCommand.RewardCommand.RewardId];
+                    await ProcessRewardCommand(role, rewardCommand);
+                }
+                else
+                {
+                    Program.Log.Error($"RoleID not found on guild: {rewardCommand.RewardCommand.RewardId}");
+                }
+            }
+        }
+
+        private async Task ProcessRewardCommand(RoleReward role, UserReward reward)
+        {
+            foreach (var user in reward.Users)
+            {
+                await GiveReward(role, user);
+            }
+        }
+
+        private async Task GiveReward(RoleReward role, UserData user)
+        {
+            if (!users.ContainsKey(user.DiscordId))
+            {
+                Program.Log.Log($"User by id '{user.DiscordId}' not found.");
+                return;
+            }
+
+            var guildUser = users[user.DiscordId];
+
+            var alreadyHas = guildUser.RoleIds.ToArray();
+            var logMessage = $"Giving reward '{role.SocketRole.Id}' to user '{user.DiscordId}'({user.Name})[" +
+                $"alreadyHas:{string.Join(",", alreadyHas.Select(a => a.ToString()))}]: ";
+
+
+            if (alreadyHas.Any(r => r == role.Reward.RoleId))
+            {
+                logMessage += "Already has role";
+                Program.Log.Log(logMessage);
+                return;
+            }
+
+            await GiveRole(guildUser, role.SocketRole);
+            await SendNotification(role, user, guildUser);
+            await Task.Delay(1000);
+            logMessage += "Role given. Notification sent.";
+            Program.Log.Log(logMessage);
+        }
+
+        private async Task GiveRole(IGuildUser user, SocketRole role)
+        {
+            try
+            {
+                Program.Log.Log($"Giving role {role.Name}={role.Id} to user {user.DisplayName}");
+                await user.AddRoleAsync(role);
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error($"Failed to give role '{role.Name}' to user '{user.DisplayName}': {ex}");
+            }
+        }
+
+        private async Task SendNotification(RoleReward reward, UserData userData, IGuildUser user)
+        {
+            try
+            {
+                if (userData.NotificationsEnabled && rewardsChannel != null)
+                {
+                    var msg = reward.Reward.Message.Replace(RewardConfig.UsernameTag, $"<@{user.Id}>");
+                    await rewardsChannel.SendMessageAsync(msg);
+                }
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error($"Failed to notify user '{user.DisplayName}' about role '{reward.SocketRole.Name}': {ex}");
+            }
+        }
+    }
+}

Tools/BiblioTech/Rewards/RewardController.cs

@@ -0,0 +1,39 @@
+using DiscordRewards;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BiblioTech.Rewards
+{
+    public interface IDiscordRoleDriver
+    {
+        Task GiveRewards(GiveRewardsCommand rewards);
+    }
+
+    [Route("api/[controller]")]
+    [ApiController]
+    public class RewardController : ControllerBase
+    {
+        [HttpGet]
+        public string Ping()
+        {
+            return "Pong";
+        }
+
+        [HttpPost]
+        public async Task<string> Give(GiveRewardsCommand cmd)
+        {
+            try
+            {
+                if (cmd.Averages != null && cmd.Averages.Any())
+                {
+                    Program.Averages = cmd.Averages;
+                }
+                await Program.RoleDriver.GiveRewards(cmd);
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error("Exception: " + ex);
+            }
+            return "OK";
+        }
+    }
+}

Tools/BiblioTech/Rewards/RewardsApi.cs

@@ -1,92 +0,0 @@
-using DiscordRewards;
-using Newtonsoft.Json;
-using System.Net;
-using TaskFactory = Utils.TaskFactory;
-
-namespace BiblioTech.Rewards
-{
-    public interface IDiscordRoleController
-    {
-        Task GiveRewards(GiveRewardsCommand rewards);
-    }
-
-    public class RewardsApi
-    {
-        private readonly HttpListener listener = new HttpListener();
-        private readonly TaskFactory taskFactory = new TaskFactory();
-        private readonly IDiscordRoleController roleController;
-        private CancellationTokenSource cts = new CancellationTokenSource();
-
-        public RewardsApi(IDiscordRoleController roleController)
-        {
-            this.roleController = roleController;
-        }
-
-        public void Start()
-        {
-            cts = new CancellationTokenSource();
-            listener.Prefixes.Add($"http://*:31080/");
-            listener.Start();
-            taskFactory.Run(ConnectionDispatcher, nameof(ConnectionDispatcher));
-        }
-
-        public void Stop()
-        {
-            listener.Stop();
-            cts.Cancel();
-            taskFactory.WaitAll();
-        }
-
-        private void ConnectionDispatcher()
-        {
-            while (!cts.Token.IsCancellationRequested)
-            {
-                var wait = listener.GetContextAsync();
-                wait.Wait(cts.Token);
-                if (wait.IsCompletedSuccessfully)
-                {
-                    taskFactory.Run(() =>
-                    {
-                        var context = wait.Result;
-                        try
-                        {
-                            HandleConnection(context).Wait();
-                        }
-                        catch (Exception ex)
-                        {
-                            Program.Log.Error("Exception during HTTP handler: " + ex);
-                        }
-                        // Whatever happens, everything's always OK.
-                        context.Response.StatusCode = 200;
-                        context.Response.OutputStream.Close();
-                    }, nameof(HandleConnection));
-                }
-            }
-        }
-
-        private async Task HandleConnection(HttpListenerContext context)
-        {
-            using var reader = new StreamReader(context.Request.InputStream);
-            var content = reader.ReadToEnd();
-
-            if (content == "Ping")
-            {
-                using var writer = new StreamWriter(context.Response.OutputStream);
-                writer.Write("Pong");
-                return;
-            }
-
-            if (!content.StartsWith("{")) return;
-            var rewards = JsonConvert.DeserializeObject<GiveRewardsCommand>(content);
-            if (rewards != null)
-            {
-                await ProcessRewards(rewards);
-            }
-        }
-
-        private async Task ProcessRewards(GiveRewardsCommand rewards)
-        {
-            await roleController.GiveRewards(rewards);
-        }
-    }
-}

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -1,26 +1,38 @@
 using Discord;
 using Discord.WebSocket;
 using DiscordRewards;
+using Newtonsoft.Json;
 
 namespace BiblioTech.Rewards
 {
-    public class RoleController : IDiscordRoleController
+    public class RoleDriver : IDiscordRoleDriver
     {
         private readonly DiscordSocketClient client;
         private readonly SocketTextChannel? rewardsChannel;
+        private readonly SocketTextChannel? eventsChannel;
         private readonly RewardRepo repo = new RewardRepo();
 
-        public RoleController(DiscordSocketClient client)
+        public RoleDriver(DiscordSocketClient client)
         {
             this.client = client;
 
-            if (!string.IsNullOrEmpty(Program.Config.RewardsChannelName))
-            {
-                rewardsChannel = GetGuild().TextChannels.SingleOrDefault(c => c.Name == Program.Config.RewardsChannelName);
-            }
+            rewardsChannel = GetChannel(Program.Config.RewardsChannelName);
+            eventsChannel = GetChannel(Program.Config.ChainEventsChannelName);
         }
 
         public async Task GiveRewards(GiveRewardsCommand rewards)
+        {
+            Program.Log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'");
+
+            if (rewards.Rewards.Any())
+            {
+                await ProcessRewards(rewards);
+            }
+
+            await ProcessChainEvents(rewards.EventsOverview);
+        }
+
+        private async Task ProcessRewards(GiveRewardsCommand rewards)
         {
             var guild = GetGuild();
             // We load all role and user information first,
@@ -33,8 +45,31 @@ namespace BiblioTech.Rewards
             await context.ProcessGiveRewardsCommand(LookUpUsers(rewards));
         }
 
+        private SocketTextChannel? GetChannel(string name)
+        {
+            if (string.IsNullOrEmpty(name)) return null;
+            return GetGuild().TextChannels.SingleOrDefault(c => c.Name == name);
+        }
+
+        private async Task ProcessChainEvents(string[] eventsOverview)
+        {
+            if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
+            await Task.Run(async () =>
+            {
+                foreach (var e in eventsOverview)
+                {
+                    if (!string.IsNullOrEmpty(e))
+                    {
+                        await eventsChannel.SendMessageAsync(e);
+                        await Task.Delay(3000);
+                    }
+                }
+            });
+        }
+
         private async Task<Dictionary<ulong, IGuildUser>> LoadAllUsers(SocketGuild guild)
         {
+            Program.Log.Log("Loading all users:");
             var result = new Dictionary<ulong, IGuildUser>();
             var users = guild.GetUsersAsync();
             await foreach (var ulist in users)
@@ -42,6 +77,8 @@ namespace BiblioTech.Rewards
                 foreach (var u in ulist)
                 {
                     result.Add(u.Id, u);
+                    var roleIds = string.Join(",", u.RoleIds.Select(r => r.ToString()).ToArray());
+                    Program.Log.Log($" > {u.Id}({u.DisplayName}) has [{roleIds}]");
                 }
             }
             return result;
@@ -96,7 +133,10 @@ namespace BiblioTech.Rewards
         {
             try
             {
-                return Program.UserRepo.GetUserDataForAddress(new GethPlugin.EthAddress(address));
+                var userData =  Program.UserRepo.GetUserDataForAddress(new GethPlugin.EthAddress(address));
+                if (userData != null) Program.Log.Log($"User '{userData.Name}' was looked up.");
+                else Program.Log.Log($"Lookup for user was unsuccessful. EthAddress: '{address}'");
+                return userData;
             }
             catch (Exception ex)
             {
@@ -107,7 +147,13 @@ namespace BiblioTech.Rewards
 
         private SocketGuild GetGuild()
         {
-            return client.Guilds.Single(g => g.Name == Program.Config.ServerName);
+            var guild = client.Guilds.SingleOrDefault(g => g.Name == Program.Config.ServerName);
+            if (guild == null)
+            {
+                throw new Exception($"Unable to find guild by name: '{Program.Config.ServerName}'. " +
+                    $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name))}]");
+            }
+            return guild;
         }
     }
 
@@ -134,85 +180,4 @@ namespace BiblioTech.Rewards
         public RewardUsersCommand RewardCommand { get; }
         public UserData[] Users { get; }
     }
-
-    public class RewardContext
-    {
-        private readonly Dictionary<ulong, IGuildUser> users;
-        private readonly Dictionary<ulong, RoleReward> roles;
-        private readonly SocketTextChannel? rewardsChannel;
-
-        public RewardContext(Dictionary<ulong, IGuildUser> users, Dictionary<ulong, RoleReward> roles, SocketTextChannel? rewardsChannel)
-        {
-            this.users = users;
-            this.roles = roles;
-            this.rewardsChannel = rewardsChannel;
-        }
-
-        public async Task ProcessGiveRewardsCommand(UserReward[] rewards)
-        {
-            foreach (var rewardCommand in rewards)
-            {
-                if (roles.ContainsKey(rewardCommand.RewardCommand.RewardId))
-                {
-                    var role = roles[rewardCommand.RewardCommand.RewardId];
-                    await ProcessRewardCommand(role, rewardCommand);
-                }
-            }
-        }
-
-        private async Task ProcessRewardCommand(RoleReward role, UserReward reward)
-        {
-            foreach (var user in reward.Users)
-            {
-                await GiveReward(role, user);
-            }
-        }
-
-        private async Task GiveReward(RoleReward role, UserData user)
-        {
-            if (!users.ContainsKey(user.DiscordId))
-            {
-                Program.Log.Log($"User by id '{user.DiscordId}' not found.");
-                return;
-            }
-
-            var guildUser = users[user.DiscordId];
-
-            var alreadyHas = guildUser.RoleIds.ToArray();
-            if (alreadyHas.Any(r => r == role.Reward.RoleId)) return;
-
-            await GiveRole(guildUser, role.SocketRole);
-            await SendNotification(role, user, guildUser);
-            await Task.Delay(1000);
-        }
-
-        private async Task GiveRole(IGuildUser user, SocketRole role)
-        {
-            try
-            {
-                Program.Log.Log($"Giving role {role.Name}={role.Id} to user {user.DisplayName}");
-                await user.AddRoleAsync(role);
-            }
-            catch (Exception ex)
-            {
-                Program.Log.Error($"Failed to give role '{role.Name}' to user '{user.DisplayName}': {ex}");
-            }
-        }
-
-        private async Task SendNotification(RoleReward reward, UserData userData, IGuildUser user)
-        {
-            try
-            {
-                if (userData.NotificationsEnabled && rewardsChannel != null)
-                {
-                    var msg = reward.Reward.Message.Replace(RewardConfig.UsernameTag, $"<@{user.Id}>");
-                    await rewardsChannel.SendMessageAsync(msg);
-                }
-            }
-            catch (Exception ex)
-            {
-                Program.Log.Error($"Failed to notify user '{user.DisplayName}' about role '{reward.SocketRole.Name}': {ex}");
-            }
-        }
-    }
 }

Tools/BiblioTech/docker/Dockerfile

@@ -1,14 +1,26 @@
 # Variables
-ARG IMAGE=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 
-# Create
-FROM ${IMAGE}
+
+# Build
+FROM ${IMAGE} AS builder
 ARG APP_HOME
 
 WORKDIR ${APP_HOME}
 COPY ./Tools/BiblioTech ./Tools/BiblioTech
 COPY ./Framework ./Framework
 COPY ./ProjectPlugins ./ProjectPlugins
-CMD ["dotnet", "run", "--project", "Tools/BiblioTech"]
+RUN dotnet restore Tools/BiblioTech
+RUN dotnet publish Tools/BiblioTech -c Release -o out
 
+
+# Create
+FROM ${IMAGE}
+ARG APP_HOME
+ENV APP_HOME=${APP_HOME}
+
+WORKDIR ${APP_HOME}
+COPY --from=builder ${APP_HOME}/out .
+CMD dotnet ${APP_HOME}/BiblioTech.dll

Tools/CodexNetDeployer/Configuration.cs

@@ -116,6 +116,9 @@ namespace CodexNetDeployer
 
         [Uniform("dbot-adminchannelname", "dbotacn", "DBOTADMINCHANNELNAME", false, "Required if discord-bot is true. Name of the Discord channel in which admin commands are allowed.")]
         public string DiscordBotAdminChannelName { get; set; } = string.Empty;
+        
+        [Uniform("dbot-rewardchannelname", "dbotrcn", "DBOTREWARDCHANNELNAME", false, "Required if discord-bot is true. Name of the Discord channel in which reward updates are posted.")]
+        public string DiscordBotRewardChannelName { get; set; } = string.Empty;
 
         [Uniform("dbot-datapath", "dbotdp", "DBOTDATAPATH", false, "Optional. Path in container where bot will save all data.")]
         public string DiscordBotDataPath { get; set; } = string.Empty;
@@ -163,6 +166,7 @@ namespace CodexNetDeployer
                 StringIsSet(nameof(DiscordBotServerName), DiscordBotServerName, errors);
                 StringIsSet(nameof(DiscordBotAdminRoleName), DiscordBotAdminRoleName, errors);
                 StringIsSet(nameof(DiscordBotAdminChannelName), DiscordBotAdminChannelName, errors);
+                StringIsSet(nameof(DiscordBotRewardChannelName), DiscordBotRewardChannelName, errors);
             }
 
             return errors;

Tools/CodexNetDeployer/Deployer.cs

@@ -122,7 +122,7 @@ namespace CodexNetDeployer
             });
         }
 
-        private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
+        private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
             CodexContractsDeployment contractsDeployment)
         {
             if (!config.DeployDiscordBot) return null;
@@ -145,7 +145,8 @@ namespace CodexNetDeployer
                 adminRoleName: config.DiscordBotAdminRoleName,
                 adminChannelName: config.DiscordBotAdminChannelName,
                 kubeNamespace: config.KubeNamespace,
-                gethInfo: info)
+                gethInfo: info,
+                rewardChannelName: config.DiscordBotRewardChannelName)
             {
                 DataPath = config.DiscordBotDataPath
             });
@@ -154,7 +155,7 @@ namespace CodexNetDeployer
             return rc;
         }
 
-        private RunningContainers? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
+        private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
         {
             if (!config.MetricsScraper || !startResults.Any()) return null;
 
@@ -179,7 +180,7 @@ namespace CodexNetDeployer
 
         private CodexInstance CreateCodexInstance(ICodexNode node)
         {
-            return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo());
+            return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -269,7 +270,7 @@ namespace CodexNetDeployer
             return TimeSpan.FromMinutes(10);
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return TimeSpan.FromSeconds(30);
         }

Tools/CodexNetDeployer/K8sHook.cs

@@ -18,11 +18,11 @@ namespace CodexNetDeployer
             this.metadata = metadata;
         }
 
-        public void OnContainersStarted(RunningContainers rc)
+        public void OnContainersStarted(RunningPod rc)
         {
         }
 
-        public void OnContainersStopped(RunningContainers rc)
+        public void OnContainersStopped(RunningPod rc)
         {
         }
 

Tools/TestNetRewarder/BotClient.cs

@@ -1,6 +1,6 @@
 using DiscordRewards;
 using Logging;
-using Newtonsoft.Json;
+using System.Net.Http.Json;
 
 namespace TestNetRewarder
 {
@@ -17,21 +17,41 @@ namespace TestNetRewarder
 
         public async Task<bool> IsOnline()
         {
-            return await HttpPost("Ping") == "Ping";
+            var result = await HttpGet();
+            log.Log("Is DiscordBot online: " + result);
+            return result == "Pong";
         }
 
-        public async Task SendRewards(GiveRewardsCommand command)
+        public async Task<bool> SendRewards(GiveRewardsCommand command)
         {
-            if (command == null || command.Rewards == null || !command.Rewards.Any()) return;
-            await HttpPost(JsonConvert.SerializeObject(command));
+            if (command == null) return false;
+            var result = await HttpPostJson(command);
+            log.Log("Reward response: " + result);
+            return result == "OK";
         }
 
-        private async Task<string> HttpPost(string content)
+        private async Task<string> HttpGet()
         {
             try
             {
                 var client = new HttpClient();
-                var response = await client.PostAsync(GetUrl(), new StringContent(content));
+                var response = await client.GetAsync(GetUrl());
+                return await response.Content.ReadAsStringAsync();
+            }
+            catch (Exception ex)
+            {
+                log.Error(ex.ToString());
+                return string.Empty;
+            }
+        }
+
+        private async Task<string> HttpPostJson<T>(T body)
+        {
+            try
+            {
+                using var client = new HttpClient();
+                using var content = JsonContent.Create(body);
+                using var response = await client.PostAsync(GetUrl(), content);
                 return await response.Content.ReadAsStringAsync();
             }
             catch (Exception ex)
@@ -43,7 +63,7 @@ namespace TestNetRewarder
 
         private string GetUrl()
         {
-            return $"{configuration.DiscordHost}:{configuration.DiscordPort}";
+            return $"{configuration.DiscordHost}:{configuration.DiscordPort}/api/reward";
         }
     }
 }

Tools/TestNetRewarder/ChainState.cs

@@ -1,5 +1,7 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
+using NethereumWorkflow.BlockUtils;
+using Newtonsoft.Json;
 using Utils;
 
 namespace TestNetRewarder
@@ -7,20 +9,45 @@ namespace TestNetRewarder
     public class ChainState
     {
         private readonly HistoricState historicState;
-
-        public ChainState(HistoricState historicState, ICodexContracts contracts, TimeRange timeRange)
+        private readonly string[] colorIcons = new[]
         {
-            NewRequests = contracts.GetStorageRequests(timeRange);
+            "🔴",
+            "🟠",
+            "🟡",
+            "🟢",
+            "🔵",
+            "🟣",
+            "🟤",
+            "⚫",
+            "⚪",
+            "🟥",
+            "🟧",
+            "🟨",
+            "🟩",
+            "🟦",
+            "🟪",
+            "🟫",
+            "⬛",
+            "⬜",
+            "🔶",
+            "🔷"
+        };
+
+        public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange)
+        {
+            this.historicState = historicState;
+
+            NewRequests = contracts.GetStorageRequests(blockRange);
+            historicState.CleanUpOldRequests();
             historicState.ProcessNewRequests(NewRequests);
             historicState.UpdateStorageRequests(contracts);
 
             StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray();
-            FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFininshed).ToArray();
-            RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(timeRange);
-            RequestCancelledEvents = contracts.GetRequestCancelledEvents(timeRange);
-            SlotFilledEvents = contracts.GetSlotFilledEvents(timeRange);
-            SlotFreedEvents = contracts.GetSlotFreedEvents(timeRange);
-            this.historicState = historicState;
+            FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray();
+            RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange);
+            RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange);
+            SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
+            SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
         }
 
         public Request[] NewRequests { get; }
@@ -31,5 +58,94 @@ namespace TestNetRewarder
         public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
         public SlotFilledEventDTO[] SlotFilledEvents { get; }
         public SlotFreedEventDTO[] SlotFreedEvents { get; }
+
+        public string[] GenerateOverview()
+        {
+            var entries = new List<StringBlockNumberPair>();
+
+            entries.AddRange(NewRequests.Select(ToPair));
+            entries.AddRange(RequestFulfilledEvents.Select(ToPair));
+            entries.AddRange(RequestCancelledEvents.Select(ToPair));
+            entries.AddRange(SlotFilledEvents.Select(ToPair));
+            entries.AddRange(SlotFreedEvents.Select(ToPair));
+            entries.AddRange(FinishedRequests.Select(ToPair));
+
+            entries.Sort(new StringUtcComparer());
+
+            return entries.Select(ToLine).ToArray();
+        }
+
+        private StringBlockNumberPair ToPair(Request r)
+        {
+            return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        public StringBlockNumberPair ToPair(StorageRequest r)
+        {
+            return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId);
+        }
+
+        private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r)
+        {
+            return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        private StringBlockNumberPair ToPair(RequestCancelledEventDTO r)
+        {
+            return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        private StringBlockNumberPair ToPair(SlotFilledEventDTO r)
+        {
+            return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        private StringBlockNumberPair ToPair(SlotFreedEventDTO r)
+        {
+            return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
+        }
+
+        private string ToLine(StringBlockNumberPair pair)
+        {
+            var nl = Environment.NewLine;
+            var colorIcon = GetColorIcon(pair.RequestId);
+            return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" +
+                $"```json{nl}" +
+                $"{pair.Str}{nl}" +
+                $"```";
+        }
+
+        private string GetColorIcon(byte[] requestId)
+        {
+            var index = requestId[0] % colorIcons.Length;
+            return colorIcons[index];
+        }
+
+        public class StringBlockNumberPair
+        {
+            public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId)
+            {
+                Name = name;
+                Str = str;
+                Block = block;
+                RequestId = requestId;
+            }
+
+            public string Name { get; }
+            public string Str { get; }
+            public BlockTimeEntry Block { get; }
+            public byte[] RequestId { get; }
+        }
+
+        public class StringUtcComparer : IComparer<StringBlockNumberPair>
+        {
+            public int Compare(StringBlockNumberPair? x, StringBlockNumberPair? y)
+            {
+                if (x == null && y == null) return 0;
+                if (x == null) return 1;
+                if (y == null) return -1;
+                return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber);
+            }
+        }
     }
 }

Tools/TestNetRewarder/Configuration.cs

@@ -14,11 +14,17 @@ namespace TestNetRewarder
         public int DiscordPort { get; set; } = 31080;
 
         [Uniform("interval-minutes", "im", "INTERVALMINUTES", false, "time in minutes between reward updates. (default 15)")]
-        public int Interval { get; set; } = 15;
+        public int IntervalMinutes { get; set; } = 15;
 
         [Uniform("check-history", "ch", "CHECKHISTORY", true, "Unix epoc timestamp of a moment in history on which processing begins. Required for hosting rewards. Should be 'launch of the testnet'.")]
         public int CheckHistoryTimestamp { get; set; } = 0;
 
+        [Uniform("market-insights", "mi", "MARKETINSIGHTS", false, "Semi-colon separated integers. Each represents a multiple of intervals, for which a market insights average will be generated.")]
+        public string MarketInsights { get; set; } = "1;96";
+
+        [Uniform("events-overview", "eo", "EVENTSOVERVIEW", false, "When greater than zero, chain event summary will be generated. (default 1)")]
+        public int CreateChainEventsOverview { get; set; } = 1;
+
         public string LogPath
         {
             get
@@ -26,5 +32,13 @@ namespace TestNetRewarder
                 return Path.Combine(DataPath, "logs");
             }
         }
+
+        public TimeSpan Interval
+        {
+            get
+            {
+                return TimeSpan.FromMinutes(IntervalMinutes);
+            }
+        }
     }
 }

Tools/TestNetRewarder/HistoricState.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
+using Newtonsoft.Json;
 
 namespace TestNetRewarder
 {
@@ -19,6 +20,15 @@ namespace TestNetRewarder
         {
             foreach (var r in storageRequests) r.Update(contracts);
         }
+
+        public void CleanUpOldRequests()
+        {
+            storageRequests.RemoveAll(r =>
+                r.State == RequestState.Cancelled ||
+                r.State == RequestState.Finished ||
+                r.State == RequestState.Failed
+            );
+        }
     }
 
     public class StorageRequest
@@ -32,12 +42,16 @@ namespace TestNetRewarder
         public Request Request { get; }
         public EthAddress[] Hosts { get; private set; }
         public RequestState State { get; private set; }
+        
+        [JsonIgnore]
         public bool RecentlyStarted { get; private set; }
-        public bool RecentlyFininshed { get; private set; }
+
+        [JsonIgnore]
+        public bool RecentlyFinished { get; private set; }
 
         public void Update(ICodexContracts contracts)
         {
-            Hosts = GetHosts(contracts);
+            var newHosts = GetHosts(contracts);
 
             var newState = contracts.GetRequestState(Request);
 
@@ -45,11 +59,12 @@ namespace TestNetRewarder
                 State == RequestState.New &&
                 newState == RequestState.Started;
 
-            RecentlyFininshed =
+            RecentlyFinished =
                 State == RequestState.Started &&
                 newState == RequestState.Finished;
 
             State = newState;
+            Hosts = newHosts;
         }
 
         private EthAddress[] GetHosts(ICodexContracts contracts)

Tools/TestNetRewarder/MarketTracker.cs

@@ -0,0 +1,124 @@
+using CodexContractsPlugin.Marketplace;
+using DiscordRewards;
+using System.Numerics;
+
+namespace TestNetRewarder
+{
+    public class MarketTracker
+    {
+        private readonly List<ChainState> buffer = new List<ChainState>();
+
+        public MarketAverage[] ProcessChainState(ChainState chainState)
+        {
+            var intervalCounts = GetInsightCounts();
+            if (!intervalCounts.Any()) return Array.Empty<MarketAverage>();
+
+            UpdateBuffer(chainState, intervalCounts.Max());
+            var result = intervalCounts
+                .Select(GenerateMarketAverage)
+                .Where(a => a != null)
+                .Cast<MarketAverage>()
+                .ToArray();
+
+            if (!result.Any()) result = Array.Empty<MarketAverage>();
+            return result;
+        }
+
+        private void UpdateBuffer(ChainState chainState, int maxNumberOfIntervals)
+        {
+            buffer.Add(chainState);
+            while (buffer.Count > maxNumberOfIntervals)
+            {
+                buffer.RemoveAt(0);
+            }
+        }
+
+        private MarketAverage? GenerateMarketAverage(int numberOfIntervals)
+        {
+            var states = SelectStates(numberOfIntervals);
+            return CreateAverage(states);
+        }
+
+        private ChainState[] SelectStates(int numberOfIntervals)
+        {
+            if (numberOfIntervals < 1) return Array.Empty<ChainState>();
+            if (numberOfIntervals > buffer.Count) return Array.Empty<ChainState>();
+            return buffer.TakeLast(numberOfIntervals).ToArray();
+        }
+
+        private MarketAverage? CreateAverage(ChainState[] states)
+        {
+            try
+            {
+                return new MarketAverage
+                {
+                    NumberOfFinished = CountNumberOfFinishedRequests(states),
+                    TimeRangeSeconds = GetTotalTimeRange(states),
+                    Price = Average(states, s => s.Request.Ask.Reward),
+                    Duration = Average(states, s => s.Request.Ask.Duration),
+                    Size = Average(states, s => GetTotalSize(s.Request.Ask)),
+                    Collateral = Average(states, s => s.Request.Ask.Collateral),
+                    ProofProbability = Average(states, s => s.Request.Ask.ProofProbability)
+                };
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error($"Exception in CreateAverage: {ex}");
+                return null;
+            }
+        }
+
+        private int GetTotalSize(Ask ask)
+        {
+            var nSlots = Convert.ToInt32(ask.Slots);
+            var slotSize = Convert.ToInt32(ask.SlotSize);
+            return nSlots * slotSize;
+        }
+
+        private float Average(ChainState[] states, Func<StorageRequest, BigInteger> getValue)
+        {
+            return Average(states, s => Convert.ToInt32(getValue(s)));
+        }
+
+        private float Average(ChainState[] states, Func<StorageRequest, int> getValue)
+        {
+            var sum = 0.0f;
+            var count = 0.0f;
+            foreach (var state in states)
+            {
+                foreach (var finishedRequest in state.FinishedRequests)
+                {
+                    sum += getValue(finishedRequest);
+                    count++;
+                }
+            }
+
+            if (count < 1.0f) return 0.0f;
+            return sum / count;
+        }
+
+        private int GetTotalTimeRange(ChainState[] states)
+        {
+            return Convert.ToInt32((Program.Config.Interval * states.Length).TotalSeconds);
+        }
+
+        private int CountNumberOfFinishedRequests(ChainState[] states)
+        {
+            return states.Sum(s => s.FinishedRequests.Length);
+        }
+
+        private int[] GetInsightCounts()
+        {
+            try
+            {
+                var tokens = Program.Config.MarketInsights.Split(';').ToArray();
+                return tokens.Select(t => Convert.ToInt32(t)).ToArray();
+            }
+            catch (Exception ex)
+            {
+                Program.Log.Error($"Exception when parsing MarketInsights config parameters: {ex}");
+            }
+            return Array.Empty<int>();            
+        }
+    }
+}

Tools/TestNetRewarder/Processor.cs

@@ -10,31 +10,53 @@ namespace TestNetRewarder
     {
         private static readonly HistoricState historicState = new HistoricState();
         private static readonly RewardRepo rewardRepo = new RewardRepo();
+        private static readonly MarketTracker marketTracker = new MarketTracker();
         private readonly ILog log;
+        private BlockInterval? lastBlockRange;
 
         public Processor(ILog log)
         {
             this.log = log;
         }
 
-        public async Task ProcessTimeSegment(TimeRange range)
+        public async Task ProcessTimeSegment(TimeRange timeRange)
         {
+            var connector = GethConnector.GethConnector.Initialize(log);
+            if (connector == null) throw new Exception("Invalid Geth information");
+
             try
             {
-                var connector = GethConnector.GethConnector.Initialize(log);
-                if (connector == null) return;
-
-                var chainState = new ChainState(historicState, connector.CodexContracts, range);
-                await ProcessTimeSegment(chainState);
+                var blockRange = connector.GethNode.ConvertTimeRangeToBlockRange(timeRange);
+                if (!IsNewBlockRange(blockRange))
+                {
+                    log.Log($"Block range {blockRange} was previously processed. Skipping...");
+                    return;
+                }
 
+                var chainState = new ChainState(historicState, connector.CodexContracts, blockRange);
+                await ProcessChainState(chainState);
             }
             catch (Exception ex)
             {
                 log.Error("Exception processing time segment: " + ex);
+                throw;
             }
         }
 
-        private async Task ProcessTimeSegment(ChainState chainState)
+        private bool IsNewBlockRange(BlockInterval blockRange)
+        {
+            if (lastBlockRange == null ||
+                lastBlockRange.From != blockRange.From || 
+                lastBlockRange.To != blockRange.To)
+            {
+                lastBlockRange = blockRange;
+                return true;
+            }
+
+            return false;
+        }
+
+        private async Task ProcessChainState(ChainState chainState)
         {
             var outgoingRewards = new List<RewardUsersCommand>();
             foreach (var reward in rewardRepo.Rewards)
@@ -42,26 +64,52 @@ namespace TestNetRewarder
                 ProcessReward(outgoingRewards, reward, chainState);
             }
 
-            if (outgoingRewards.Any())
+            var marketAverages = GetMarketAverages(chainState);
+            var eventsOverview = GenerateEventsOverview(chainState);
+
+            log.Log($"Found {outgoingRewards.Count} rewards. " +
+                $"Found {marketAverages.Length} market averages. " +
+                $"Found {eventsOverview.Length} events.");
+
+            if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any())
             {
-                await SendRewardsCommand(outgoingRewards);
+                if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview))
+                {
+                    log.Error("Failed to send reward command.");
+                }
             }
         }
 
-        private async Task SendRewardsCommand(List<RewardUsersCommand> outgoingRewards)
+        private string[] GenerateEventsOverview(ChainState chainState)
+        {
+            return chainState.GenerateOverview();
+        }
+
+        private MarketAverage[] GetMarketAverages(ChainState chainState)
+        {
+            return marketTracker.ProcessChainState(chainState);
+        }
+
+        private async Task<bool> SendRewardsCommand(List<RewardUsersCommand> outgoingRewards, MarketAverage[] marketAverages, string[] eventsOverview)
         {
             var cmd = new GiveRewardsCommand
             {
-                Rewards = outgoingRewards.ToArray()
+                Rewards = outgoingRewards.ToArray(),
+                Averages = marketAverages.ToArray(),
+                EventsOverview = eventsOverview
             };
 
             log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd));
-            await Program.BotClient.SendRewards(cmd);
+            return await Program.BotClient.SendRewards(cmd);
         }
 
         private void ProcessReward(List<RewardUsersCommand> outgoingRewards, RewardConfig reward, ChainState chainState)
         {
             var winningAddresses = PerformCheck(reward, chainState);
+            foreach (var win in winningAddresses)
+            {
+                log.Log($"Address '{win.Address}' wins '{reward.Message}'");
+            }
             if (winningAddresses.Any())
             {
                 outgoingRewards.Add(new RewardUsersCommand
@@ -75,7 +123,7 @@ namespace TestNetRewarder
         private EthAddress[] PerformCheck(RewardConfig reward, ChainState chainState)
         {
             var check = GetCheck(reward.CheckConfig);
-            return check.Check(chainState);
+            return check.Check(chainState).Distinct().ToArray();
         }
 
         private ICheck GetCheck(CheckConfig config)

Tools/TestNetRewarder/Program.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using GethConnector;
 using Logging;
 using Utils;
 
@@ -12,6 +11,7 @@ namespace TestNetRewarder
         public static CancellationToken CancellationToken { get; private set; }
         public static BotClient BotClient { get; private set; } = null!;
         private static Processor processor = null!;
+        private static DateTime lastCheck = DateTime.MinValue;
 
         public static Task Main(string[] args)
         {
@@ -47,7 +47,7 @@ namespace TestNetRewarder
             {
                 await EnsureBotOnline();
                 await segmenter.WaitForNextSegment(processor.ProcessTimeSegment);
-                await Task.Delay(1000, CancellationToken);
+                await Task.Delay(100, CancellationToken);
             }
         }
 
@@ -59,11 +59,15 @@ namespace TestNetRewarder
 
             var blockNumber = gc.GethNode.GetSyncedBlockNumber();
             if (blockNumber == null || blockNumber < 1) throw new Exception("Geth connection failed.");
+            Log.Log("Geth OK. Block number: " + blockNumber);
         }
 
         private static async Task EnsureBotOnline()
         {
             var start = DateTime.UtcNow;
+            var timeSince = start - lastCheck;
+            if (timeSince.TotalSeconds < 30.0) return;
+
             while (! await BotClient.IsOnline() && !CancellationToken.IsCancellationRequested)
             {
                 await Task.Delay(5000);
@@ -76,6 +80,8 @@ namespace TestNetRewarder
                     throw new Exception(msg);
                 }
             }
+
+            lastCheck = start;
         }
 
         private static void PrintHelp()

Tools/TestNetRewarder/TimeSegmenter.cs

@@ -13,10 +13,10 @@ namespace TestNetRewarder
         {
             this.log = log;
 
-            if (configuration.Interval < 0) configuration.Interval = 15;
+            if (configuration.IntervalMinutes < 0) configuration.IntervalMinutes = 1;
             if (configuration.CheckHistoryTimestamp == 0) throw new Exception("'check-history' unix timestamp is required. Set it to the start/launch moment of the testnet.");
 
-            segmentSize = TimeSpan.FromSeconds(configuration.Interval);
+            segmentSize = configuration.Interval;
             start = DateTimeOffset.FromUnixTimeSeconds(configuration.CheckHistoryTimestamp).UtcDateTime;
 
             log.Log("Starting time segments at " + start);
@@ -31,10 +31,12 @@ namespace TestNetRewarder
             if (end > now)
             {
                 // Wait for the entire time segment to be in the past.
-                var delay = (end - now).Add(TimeSpan.FromSeconds(3));
+                var delay = end - now;
                 waited = true;
+                log.Log($"Waiting till time segment is in the past... {Time.FormatDuration(delay)}");
                 await Task.Delay(delay, Program.CancellationToken);
             }
+            await Task.Delay(TimeSpan.FromSeconds(3), Program.CancellationToken);
 
             if (Program.CancellationToken.IsCancellationRequested) return;
 

Tools/TestNetRewarder/build-docker.bat

@@ -0,0 +1,2 @@
+docker build -f docker/Dockerfile -t thatbenbierens/codex-rewardbot:initial ../..
+docker push thatbenbierens/codex-rewardbot:initial

Tools/TestNetRewarder/docker/Dockerfile

@@ -1,13 +1,26 @@
 # Variables
-ARG IMAGE=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 
-# Create
-FROM ${IMAGE}
+
+# Build
+FROM ${IMAGE} AS builder
 ARG APP_HOME
 
 WORKDIR ${APP_HOME}
 COPY ./Tools/TestNetRewarder ./Tools/TestNetRewarder
 COPY ./Framework ./Framework
 COPY ./ProjectPlugins ./ProjectPlugins
-CMD ["dotnet", "run", "--project", "Tools/TestNetRewarder"]
+RUN dotnet restore Tools/TestNetRewarder
+RUN dotnet publish Tools/TestNetRewarder -c Release -o out
+
+
+# Create
+FROM ${IMAGE}
+ARG APP_HOME
+ENV APP_HOME=${APP_HOME}
+
+WORKDIR ${APP_HOME}
+COPY --from=builder ${APP_HOME}/out .
+CMD dotnet ${APP_HOME}/TestNetRewarder.dll