Finished!

# Conflicts:
#	ProjectPlugins/CodexPlugin/CodexNode.cs
#	ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

.github/workflows/docker-keymaker.yml

@@ -0,0 +1,27 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-KeyMaker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      docker_repo: codexstorage/codex-keymaker
+    secrets: inherit
+

Framework/ArgsUniform/Assigner.cs

@@ -1,4 +1,6 @@
-using System.Reflection;
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
 
 namespace ArgsUniform
 {
@@ -103,8 +105,12 @@ namespace ArgsUniform
                     if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
                     if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
                     if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
 
-                    throw new NotSupportedException();
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
                 }
             }
         }
@@ -122,7 +128,27 @@ namespace ArgsUniform
 
         private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
         {
-            if (int.TryParse(value.ToString(), out int i))
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
             {
                 uniformProperty.SetValue(result, i);
                 return true;

Framework/Core/CoreInterface.cs

@@ -30,11 +30,11 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var file = entryPoint.Tools.GetLog().CreateSubfile();
-            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
-            var logHandler = new LogDownloadHandler(container.Name, file);
+            var msg = $"Downloading container log for '{container.Name}'";
+            entryPoint.Tools.GetLog().Log(msg);
+            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
             workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return logHandler.DownloadLog();
+            return new DownloadedLog(logHandler);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/DownloadedLog.cs

@@ -1,4 +1,5 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 
 namespace Core
 {
@@ -14,9 +15,9 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(LogFile logFile)
+        internal DownloadedLog(WriteToFileLogHandler logHandler)
         {
-            this.logFile = logFile;
+            logFile = logHandler.LogFile;
         }
 
         public void IterateLines(Action<string> action)

Framework/Core/EntryPoint.cs

@@ -38,10 +38,14 @@ namespace Core
             return new CoreInterface(this);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
         }
 
         internal T GetPlugin<T>() where T : IProjectPlugin

Framework/Core/Http.cs

@@ -7,6 +7,7 @@ namespace Core
     {
         T OnClient<T>(Func<HttpClient, T> action);
         T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
         IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
@@ -35,13 +36,19 @@ namespace Core
         }
 
         public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
         {
             var client = GetClient();
 
             return LockRetry(() =>
             {
                 return action(client);
-            }, description);
+            }, retry);
         }
 
         public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
@@ -54,11 +61,11 @@ namespace Core
             return DebugStack.GetCallerName(skipFrames: 2);
         }
 
-        private T LockRetry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, Retry retry)
         {
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
+                return retry.Run(operation);
             }
         }
 

Framework/Core/LogDownloadHandler.cs

@@ -1,28 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace Core
-{
-    internal class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly LogFile log;
-
-        internal LogDownloadHandler(string description, LogFile log)
-        {
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        internal IDownloadedLog DownloadLog()
-        {
-            return new DownloadedLog(log);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}

Framework/Core/PluginManager.cs

@@ -34,12 +34,12 @@
             return metadata;
         }
 
-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
             foreach (var pair in pairs)
             {
                 pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
             }
         }
 

Framework/Core/PluginTools.cs

@@ -6,7 +6,13 @@ namespace Core
 {
     public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
     {
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+        ITimeSet TimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
     }
 
     public interface IWorkflowTool
@@ -33,7 +39,6 @@ namespace Core
 
     internal class PluginTools : IPluginTools
     {
-        private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
         private readonly LogPrefixer log;
@@ -42,10 +47,12 @@ namespace Core
         {
             this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            this.timeSet = timeSet;
+            TimeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
+        public ITimeSet TimeSet { get; }
+
         public void ApplyLogPrefix(string prefix)
         {
             log.Prefix = prefix;
@@ -53,7 +60,7 @@ namespace Core
 
         public IHttp CreateHttp(Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(onClientCreated, timeSet);
+            return CreateHttp(onClientCreated, TimeSet);
         }
 
         public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
@@ -63,7 +70,7 @@ namespace Core
 
         public IHttp CreateHttp()
         {
-            return new Http(log, timeSet);
+            return new Http(log, TimeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
@@ -71,9 +78,9 @@ namespace Core
             return workflowCreator.CreateWorkflow(namespaceOverride);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
             if (deleteTrackedFiles) fileManager.DeleteAllFiles();
         }
 

Framework/DiscordRewards/CheckConfig.cs

@@ -13,9 +13,9 @@ namespace DiscordRewards
     public enum CheckType
     {
         Uninitialized,
-        FilledSlot,
-        FinishedSlot,
-        PostedContract,
-        StartedContract,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
     }
 }

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -5,6 +5,11 @@
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
         public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
         public string[] EventsOverview { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
+        }
     }
 
     public class RewardUsersCommand

Framework/DiscordRewards/RewardRepo.cs

@@ -11,19 +11,19 @@ namespace DiscordRewards
             // Filled any slot
             new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
             {
-                Type = CheckType.FilledSlot
+                Type = CheckType.HostFilledSlot
             }),
 
             // Finished any slot
             new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
             {
-                Type = CheckType.FinishedSlot
+                Type = CheckType.HostFinishedSlot
             }),
 
             // Finished a sizable slot
             new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
             {
-                Type = CheckType.FinishedSlot,
+                Type = CheckType.HostFinishedSlot,
                 MinSlotSize = 10.MB(),
                 MinDuration = TimeSpan.FromMinutes(5.0),
             }),
@@ -31,19 +31,19 @@ namespace DiscordRewards
             // Posted any contract
             new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
             {
-                Type = CheckType.PostedContract
+                Type = CheckType.ClientPostedContract
             }),
 
             // Started any contract
             new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
             {
-                Type = CheckType.StartedContract
+                Type = CheckType.ClientStartedContract
             }),
 
             // Started a sizable contract
             new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
             {
-                Type = CheckType.StartedContract,
+                Type = CheckType.ClientStartedContract,
                 MinNumberOfHosts = 4,
                 MinSlotSize = 10.MB(),
                 MinDuration = TimeSpan.FromMinutes(5.0),

Framework/KubernetesWorkflow/ByteSizeExtensions.cs

@@ -1,41 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public static class ByteSizeExtensions
-    {
-        public static string ToSuffixNotation(this ByteSize b)
-        {
-            long x = 1024;
-            var map = new Dictionary<long, string>
-            {
-                { Pow(x, 4), "Ti" },
-                { Pow(x, 3), "Gi" },
-                { Pow(x, 2), "Mi" },
-                { (x), "Ki" },
-            };
-
-            var bytes = b.SizeInBytes;
-            foreach (var pair in map)
-            {
-                if (bytes > pair.Key)
-                {
-                    double bytesD = bytes;
-                    double divD = pair.Key;
-                    double numD = Math.Ceiling(bytesD / divD);
-                    var v = Convert.ToInt64(numD);
-                    return $"{v}{pair.Value}";
-                }
-            }
-
-            return $"{bytes}";
-        }
-
-        private static long Pow(long x, int v)
-        {
-            long result = 1;
-            for (var i = 0; i < v; i++) result *= x;
-            return result;
-        }
-    }
-}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,7 +11,6 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
-        private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
@@ -27,11 +26,10 @@ namespace KubernetesWorkflow
             cts = new CancellationTokenSource();
         }
 
-        public void Start(ILogHandler logHandler)
+        public void Start()
         {
             if (worker != null) throw new InvalidOperationException();
 
-            this.logHandler = logHandler;
             cts = new CancellationTokenSource();
             worker = Task.Run(Worker);
         }
@@ -50,7 +48,9 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client);
+            var result = HasContainerBeenRestarted(client);
+            if (result) DownloadCrashedContainerLogs(client);
+            return result;
         }
 
         private void Worker()
@@ -83,14 +83,16 @@ namespace KubernetesWorkflow
         private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result) log.Log("Pod crash detected for " + containerName);
+            return result;
         }
 
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
-            log.Log("Pod crash detected for " + containerName);
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            logHandler!.Log(stream);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            handler.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         {
             var config = GetConfig();
             UpdateHostAddress(config);
+            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
             return config;
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -45,7 +45,7 @@ namespace KubernetesWorkflow
 
         public void WaitUntilOnline(RunningContainer container)
         {
-            WaitUntilDeploymentOnline(container.Recipe.Name);
+            WaitUntilDeploymentOnline(container);
         }
 
         public PodInfo GetPodInfo(RunningDeployment deployment)
@@ -64,14 +64,14 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
         {
             log.Debug();
 
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
             logHandler.Log(stream);
         }
 
@@ -115,7 +115,7 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteAllNamespacesStartingWith(string prefix)
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
         {
             log.Debug();
 
@@ -124,25 +124,28 @@ namespace KubernetesWorkflow
 
             foreach (var ns in namespaces)
             {
-                DeleteNamespace(ns);
+                DeleteNamespace(ns, wait);
             }
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(K8sNamespace))
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
             }
         }
 
-        public void DeleteNamespace(string ns)
+        public void DeleteNamespace(string ns, bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(ns))
             {
                 client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
             }
         }
 
@@ -532,7 +535,7 @@ namespace KubernetesWorkflow
             }
             if (set.Memory.SizeInBytes != 0)
             {
-                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
             }
             return result;
         }
@@ -871,15 +874,44 @@ namespace KubernetesWorkflow
             WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
-        private void WaitUntilDeploymentOnline(string deploymentName)
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
+        private void WaitUntilDeploymentOnline(RunningContainer container)
         {
             WaitUntil(() =>
             {
-                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
+                CheckForCrash(container);
+
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
             }, nameof(WaitUntilDeploymentOnline));
         }
 
+        private void CheckForCrash(RunningContainer container)
+        {
+            var deploymentName = container.Recipe.Name;
+            var podName = GetPodName(container);
+
+            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
+            if (podInfo == null) return;
+            if (podInfo.Status == null) return;
+            if (podInfo.Status.ContainerStatuses == null) return;
+
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result)
+            {
+                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
+                log.Error(msg);
+
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+
+                throw new Exception(msg);
+            }
+        }
+
         private void WaitUntilDeploymentOffline(string deploymentName)
         {
             WaitUntil(() =>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Logging;
+
+namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
@@ -20,4 +22,25 @@
 
         protected abstract void ProcessLine(string line);
     }
+
+    public class WriteToFileLogHandler : LogHandler, ILogHandler
+    {
+        public WriteToFileLogHandler(ILog sourceLog, string description)
+        {
+            LogFile = sourceLog.CreateSubfile();
+
+            var msg = $"{description} -->> {LogFile.FullFilename}";
+            sourceLog.Log(msg);
+
+            LogFile.Write(msg);
+            LogFile.WriteRaw(description);
+        }
+
+        public LogFile LogFile { get; }
+
+        protected override void ProcessLine(string line)
+        {
+            LogFile.WriteRaw(line);
+        }
+    }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -105,7 +105,7 @@ namespace KubernetesWorkflow.Recipe
 
         protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
         {
-            var size = 10.MB().ToSuffixNotation();
+            var size = 10.MB().SizeInBytes.ToString();
             volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
         }
 
@@ -114,7 +114,7 @@ namespace KubernetesWorkflow.Recipe
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                resourceQuantity: volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.SizeInBytes.ToString()));
         }
 
         protected void Additional(object userData)

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,8 +1,5 @@
-using k8s;
-using k8s.Models;
-using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Recipe;
 using KubernetesWorkflow.Types;
-using Newtonsoft.Json;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -15,10 +15,10 @@ namespace KubernetesWorkflow
         PodInfo GetPodInfo(RunningPod pod);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
-        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace();
-        void DeleteNamespacesStartingWith(string namespacePrefix);
+        void DeleteNamespace(bool wait);
+        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
     }
 
     public class StartupWorkflow : IStartupWorkflow
@@ -106,11 +106,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
         }
 
@@ -122,19 +122,19 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteNamespace();
+                controller.DeleteNamespace(wait);
             });
         }
 
-        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
             });
         }
 

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -46,7 +46,7 @@ namespace NethereumWorkflow.BlockUtils
 
         private ulong Log(Func<ulong> operation)
         {
-            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder));
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
             var result = operation();
             sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");
 

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -117,6 +117,7 @@ namespace NethereumWorkflow
             }
 
             return new BlockInterval(
+                timeRange: timeRange,
                 from: fromBlock.Value,
                 to: toBlock.Value
             );

Framework/Utils/BlockInterval.cs

@@ -2,7 +2,7 @@
 {
     public class BlockInterval
     {
-        public BlockInterval(ulong from, ulong to)
+        public BlockInterval(TimeRange timeRange, ulong from, ulong to)
         {
             if (from < to)
             {
@@ -14,10 +14,13 @@
                 From = to;
                 To = from;
             }
+            TimeRange = timeRange;
         }
 
         public ulong From { get; }
         public ulong To { get; }
+        public TimeRange TimeRange { get; }
+        public ulong NumberOfBlocks => To - From;
 
         public override string ToString()
         {

Framework/Utils/Retry.cs

@@ -0,0 +1,131 @@
+namespace Utils
+{
+    public class Retry
+    {
+        private readonly string description;
+        private readonly TimeSpan maxTimeout;
+        private readonly TimeSpan sleepAfterFail;
+        private readonly Action<Failure> onFail;
+
+        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+        {
+            this.description = description;
+            this.maxTimeout = maxTimeout;
+            this.sleepAfterFail = sleepAfterFail;
+            this.onFail = onFail;
+        }
+
+        public void Run(Action task)
+        {
+            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+        }
+
+        public T Run<T>(Func<T> task)
+        {
+            T? result = default;
+
+            var run = new RetryRun(description, () =>
+            {
+                result = task();
+            }, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+
+            return result!;
+        }
+
+        private class RetryRun
+        {
+            private readonly string description;
+            private readonly Action task;
+            private readonly TimeSpan maxTimeout;
+            private readonly TimeSpan sleepAfterFail;
+            private readonly Action<Failure> onFail;
+            private readonly DateTime start = DateTime.UtcNow;
+            private readonly List<Failure> failures = new List<Failure>();
+            private int tryNumber;
+            private DateTime tryStart;
+
+            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+            {
+                this.description = description;
+                this.task = task;
+                this.maxTimeout = maxTimeout;
+                this.sleepAfterFail = sleepAfterFail;
+                this.onFail = onFail;
+
+                tryNumber = 0;
+                tryStart = DateTime.UtcNow;
+            }
+
+            public void Run()
+            {
+                while (true)
+                {
+                    CheckMaximums();
+
+                    tryNumber++;
+                    tryStart = DateTime.UtcNow;
+                    try
+                    {
+                        task();
+                        return;
+                    }
+                    catch (Exception ex)
+                    {
+                        var failure = CaptureFailure(ex);
+                        onFail(failure);
+                        Time.Sleep(sleepAfterFail);
+                    }
+                }
+            }
+
+            private Failure CaptureFailure(Exception ex)
+            {
+                var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
+                failures.Add(f);
+                return f;
+            }
+
+            private void CheckMaximums()
+            {
+                if (Duration() > maxTimeout) Fail();
+            }
+
+            private void Fail()
+            {
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                        new AggregateException(failures.Select(f => f.Exception)));
+            }
+
+            private string GetFailureReport()
+            {
+                return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
+            }
+
+            private TimeSpan Duration()
+            {
+                return DateTime.UtcNow - start;
+            }
+        }
+    }
+
+    public class Failure
+    {
+        public Failure(Exception exception, TimeSpan duration, int tryNumber)
+        {
+            Exception = exception;
+            Duration = duration;
+            TryNumber = tryNumber;
+        }
+
+        public Exception Exception { get; }
+        public TimeSpan Duration { get; }
+        public int TryNumber { get; }
+
+        public string Describe()
+        {
+            return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
+        }
+    }
+}

Framework/Utils/Time.cs

@@ -1,6 +1,4 @@
-using System.Diagnostics;
-
-namespace Utils
+namespace Utils
 {
     public static class Time
     {
@@ -20,6 +18,12 @@ namespace Utils
             task.Wait();
         }
 
+        public static string FormatDuration(TimeSpan? d)
+        {
+            if (d == null) return "[NULL]";
+            return FormatDuration(d.Value);
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -105,78 +109,24 @@ namespace Utils
 
         public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var tries = 1;
-            var tryInfo = new List<(Exception, TimeSpan)>();
-
-            while (true)
-            {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
-                {
-                    var info = FormatTryInfos(tryInfo);
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
-                }
-
-                var sw = Stopwatch.StartNew();
-                try
-                {
-                    action();
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    tryInfo.Add((ex, sw.Elapsed));
-                    tries++;
-                }
-
-                Sleep(retryTime);
-            }
-        }
-
-        private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
-        {
-            return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
-        }
-
-        private static string FormatTryInfo((Exception, TimeSpan) info, int index)
-        {
-            return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
-        }
-
-        private static Action<int> failedCallback = i => { };
-        public static void SetRetryFailedCallback(Action<int> onRetryFailed)
-        {
-            failedCallback = onRetryFailed;
+            Retry(action, maxTimeout, retryTime, description, f => { });
         }
 
         public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
-            var start = DateTime.UtcNow;
-            var tries = 1;
-            var exceptions = new List<Exception>();
+            return Retry(action, maxTimeout, retryTime, description, f => { });
+        }
 
-            while (true)
-            {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
-                {
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            r.Run(action);
+        }
 
-                try
-                {
-                    return action();
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    failedCallback(tries);
-                    tries++;
-                }
-
-                Sleep(retryTime);
-            }
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            return r.Run(action);
         }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -0,0 +1,68 @@
+using CodexContractsPlugin.Marketplace;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainEvents
+    {
+        private ChainEvents(
+            BlockInterval blockInterval,
+            Request[] requests,
+            RequestFulfilledEventDTO[] fulfilled,
+            RequestCancelledEventDTO[] cancelled,
+            SlotFilledEventDTO[] slotFilled,
+            SlotFreedEventDTO[] slotFreed
+            )
+        {
+            BlockInterval = blockInterval;
+            Requests = requests;
+            Fulfilled = fulfilled;
+            Cancelled = cancelled;
+            SlotFilled = slotFilled;
+            SlotFreed = slotFreed;
+        }
+
+        public BlockInterval BlockInterval { get; }
+        public Request[] Requests { get; }
+        public RequestFulfilledEventDTO[] Fulfilled { get; }
+        public RequestCancelledEventDTO[] Cancelled { get; }
+        public SlotFilledEventDTO[] SlotFilled { get; }
+        public SlotFreedEventDTO[] SlotFreed { get; }
+
+        public IHasBlock[] All
+        {
+            get
+            {
+                var all = new List<IHasBlock>();
+                all.AddRange(Requests);
+                all.AddRange(Fulfilled);
+                all.AddRange(Cancelled);
+                all.AddRange(SlotFilled);
+                all.AddRange(SlotFreed);
+                return all.ToArray();
+            }
+        }
+
+        public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
+        {
+            return FromContractEvents(contracts.GetEvents(blockInterval));
+        }
+
+        public static ChainEvents FromTimeRange(ICodexContracts contracts, TimeRange timeRange)
+        {
+            return FromContractEvents(contracts.GetEvents(timeRange));
+        }
+
+        public static ChainEvents FromContractEvents(ICodexContractsEvents events)
+        {
+            return new ChainEvents(
+                events.BlockInterval,
+                events.GetStorageRequests(),
+                events.GetRequestFulfilledEvents(),
+                events.GetRequestCancelledEvents(),
+                events.GetSlotFilledEvents(),
+                events.GetSlotFreedEvents()
+            );
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -0,0 +1,156 @@
+using CodexContractsPlugin.Marketplace;
+using Logging;
+using System.Numerics;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateChangeHandler
+    {
+        void OnNewRequest(IChainStateRequest request);
+        void OnRequestFinished(IChainStateRequest request);
+        void OnRequestFulfilled(IChainStateRequest request);
+        void OnRequestCancelled(IChainStateRequest request);
+        void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex);
+        void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex);
+    }
+
+    public class ChainState
+    {
+        private readonly List<ChainStateRequest> requests = new List<ChainStateRequest>();
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly IChainStateChangeHandler handler;
+
+        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
+        {
+            this.log = new LogPrefixer(log, "(ChainState) ");
+            this.contracts = contracts;
+            handler = changeHandler;
+            StartUtc = startUtc;
+            TotalSpan = new TimeRange(startUtc, startUtc);
+        }
+
+        public TimeRange TotalSpan { get; private set; }
+        public IChainStateRequest[] Requests => requests.ToArray();
+
+        public DateTime StartUtc { get; }
+
+        public void Update()
+        {
+            Update(DateTime.UtcNow);
+        }
+
+        public void Update(DateTime toUtc)
+        {
+            var span = new TimeRange(TotalSpan.To, toUtc);
+            var events = ChainEvents.FromTimeRange(contracts, span);
+            Apply(events);
+
+            TotalSpan = new TimeRange(TotalSpan.From, span.To);
+        }
+
+        private void Apply(ChainEvents events)
+        {
+            if (events.BlockInterval.TimeRange.From < TotalSpan.From)
+                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+
+            log.Log($"ChainState updating: {events.BlockInterval}");
+
+            // Run through each block and apply the events to the state in order.
+            var span = events.BlockInterval.TimeRange.Duration;
+            var numBlocks = events.BlockInterval.NumberOfBlocks;
+            var spanPerBlock = span / numBlocks;
+
+            var eventUtc = events.BlockInterval.TimeRange.From;
+            for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++)
+            {
+                var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
+                ApplyEvents(b, blockEvents, eventUtc);
+
+                eventUtc += spanPerBlock;
+            }
+        }
+
+        private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
+        {
+            foreach (var e in blockEvents)
+            {
+                dynamic d = e;
+                ApplyEvent(d);
+            }
+
+            ApplyTimeImplicitEvents(blockNumber, eventsUtc);
+        }
+
+        private void ApplyEvent(Request request)
+        {
+            if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId)))
+                throw new Exception("Received NewRequest event for id that already exists.");
+
+            var newRequest = new ChainStateRequest(log, request, RequestState.New);
+            requests.Add(newRequest);
+
+            handler.OnNewRequest(newRequest);
+        }
+
+        private void ApplyEvent(RequestFulfilledEventDTO request)
+        {
+            var r = FindRequest(request.RequestId);
+            if (r == null) return;
+            r.UpdateState(request.Block.BlockNumber, RequestState.Started);
+            handler.OnRequestFulfilled(r);
+        }
+
+        private void ApplyEvent(RequestCancelledEventDTO request)
+        {
+            var r = FindRequest(request.RequestId);
+            if (r == null) return;
+            r.UpdateState(request.Block.BlockNumber, RequestState.Cancelled);
+            handler.OnRequestCancelled(r);
+        }
+
+        private void ApplyEvent(SlotFilledEventDTO request)
+        {
+            var r = FindRequest(request.RequestId);
+            if (r == null) return;
+            r.Hosts.Add(request.Host, (int)request.SlotIndex);
+            r.Log($"[{request.Block.BlockNumber}] SlotFilled (host:'{request.Host}', slotIndex:{request.SlotIndex})");
+            handler.OnSlotFilled(r, request.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotFreedEventDTO request)
+        {
+            var r = FindRequest(request.RequestId);
+            if (r == null) return;
+            r.Hosts.RemoveHost((int)request.SlotIndex);
+            r.Log($"[{request.Block.BlockNumber}] SlotFreed (slotIndex:{request.SlotIndex})");
+            handler.OnSlotFreed(r, request.SlotIndex);
+        }
+
+        private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
+        {
+            foreach (var r in requests)
+            {
+                if (r.State == RequestState.Started
+                    && r.FinishedUtc < eventsUtc)
+                {
+                    r.UpdateState(blockNumber, RequestState.Finished);
+                    handler.OnRequestFinished(r);
+                }
+            }
+        }
+
+        private ChainStateRequest? FindRequest(byte[] requestId)
+        {
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
+            if (r == null) log.Log("Unable to find request by ID!");
+            return r;
+        }
+
+        private bool Equal(byte[] a, byte[] b)
+        {
+            return a.SequenceEqual(b);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -0,0 +1,80 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public interface IChainStateRequest
+    {
+        Request Request { get; }
+        RequestState State { get; }
+        DateTime ExpiryUtc { get; }
+        DateTime FinishedUtc { get; }
+        EthAddress Client { get; }
+        RequestHosts Hosts { get; }
+    }
+
+    public class ChainStateRequest : IChainStateRequest
+    {
+        private readonly ILog log;
+
+        public ChainStateRequest(ILog log, Request request, RequestState state)
+        {
+            this.log = log;
+            Request = request;
+            State = state;
+
+            ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry);
+            FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration);
+
+            Log($"[{request.Block.BlockNumber}] Created as {State}.");
+
+            Client = new EthAddress(request.Client);
+            Hosts = new RequestHosts();
+        }
+
+        public Request Request { get; }
+        public RequestState State { get; private set; }
+        public DateTime ExpiryUtc { get; }
+        public DateTime FinishedUtc { get; }
+        public EthAddress Client { get; }
+        public RequestHosts Hosts { get; }
+
+        public void UpdateState(ulong blockNumber, RequestState newState)
+        {
+            Log($"[{blockNumber}] Transit: {State} -> {newState}");
+            State = newState;
+        }
+
+        public void Log(string msg)
+        {
+            log.Log($"Request '{Request.Id}': {msg}");
+        }
+    }
+
+    public class RequestHosts
+    {
+        private readonly Dictionary<int, EthAddress> hosts = new Dictionary<int, EthAddress>();
+
+        public void Add(EthAddress host, int index)
+        {
+            hosts.Add(index, host);
+        }
+        
+        public void RemoveHost(int index)
+        {
+            hosts.Remove(index);
+        }
+
+        public EthAddress? GetHost(int index)
+        {
+            if (!hosts.ContainsKey(index)) return null;
+            return hosts[index];
+        }
+
+        public EthAddress[] GetHosts()
+        {
+            return hosts.Values.ToArray();
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -0,0 +1,31 @@
+using System.Numerics;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class DoNothingChainEventHandler : IChainStateChangeHandler
+    {
+        public void OnNewRequest(IChainStateRequest request)
+        {
+        }
+
+        public void OnRequestCancelled(IChainStateRequest request)
+        {
+        }
+
+        public void OnRequestFinished(IChainStateRequest request)
+        {
+        }
+
+        public void OnRequestFulfilled(IChainStateRequest request)
+        {
+        }
+
+        public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
+        {
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,10 +2,8 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
-using Nethereum.Hex.HexTypes;
 using Nethereum.Util;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Utils;
@@ -22,13 +20,10 @@ namespace CodexContractsPlugin
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
 
-        Request[] GetStorageRequests(BlockInterval blockRange);
+        ICodexContractsEvents GetEvents(TimeRange timeRange);
+        ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
-        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange);
-        RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange);
-        SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange);
-        SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange);
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -67,7 +62,7 @@ namespace CodexContractsPlugin
 
         public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.TstWei, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -78,68 +73,17 @@ namespace CodexContractsPlugin
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
             var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
-            return balance.TestTokens();
+            return balance.TstWei();
         }
 
-        public Request[] GetStorageRequests(BlockInterval blockRange)
+        public ICodexContractsEvents GetEvents(TimeRange timeRange)
         {
-            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, blockRange);
-            var i = StartInteraction();
-            return events
-                    .Select(e =>
-                    {
-                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+            return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange));
         }
 
-        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange)
+        public ICodexContractsEvents GetEvents(BlockInterval blockInterval)
         {
-            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
-        }
-
-        public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange)
-        {
-            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, blockRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange)
-        {
-            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, blockRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
-                return result;
-            }).ToArray();
-        }
-
-        public SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange)
-        {
-            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, blockRange);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
         }
 
         public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
@@ -170,17 +114,6 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
-        private BlockTimeEntry GetBlock(ulong number)
-        {
-            return gethNode.GetBlockForNumber(number);
-        }
-
-        private EthAddress GetEthAddressFromTransaction(string transactionHash)
-        {
-            var transaction = gethNode.GetTransaction(transactionHash);
-            return new EthAddress(transaction.From);
-        }
-
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -0,0 +1,108 @@
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
+using Logging;
+using Nethereum.Hex.HexTypes;
+using NethereumWorkflow.BlockUtils;
+using Utils;
+
+namespace CodexContractsPlugin
+{
+    public interface ICodexContractsEvents
+    {
+        BlockInterval BlockInterval { get; }
+        Request[] GetStorageRequests();
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
+        RequestCancelledEventDTO[] GetRequestCancelledEvents();
+        SlotFilledEventDTO[] GetSlotFilledEvents();
+        SlotFreedEventDTO[] GetSlotFreedEvents();
+    }
+
+    public class CodexContractsEvents : ICodexContractsEvents
+    {
+        private readonly ILog log;
+        private readonly IGethNode gethNode;
+        private readonly CodexContractsDeployment deployment;
+
+        public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval)
+        {
+            this.log = log;
+            this.gethNode = gethNode;
+            this.deployment = deployment;
+            BlockInterval = blockInterval;
+        }
+        
+        public BlockInterval BlockInterval { get; }
+
+        public Request[] GetStorageRequests()
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            var i = new ContractInteractions(log, gethNode);
+            return events
+            .Select(e =>
+            {
+                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents()
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents()
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents()
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
+        }
+
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            return gethNode.GetBlockForNumber(number);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -44,7 +44,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string MintTestTokens(EthAddress address, BigInteger amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
             return MintTokens(address.Address, amount, tokenAddress);
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private string MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, BigInteger amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -93,7 +93,7 @@ namespace CodexContractsPlugin
             var function = new MintTokensFunction
             {
                 Holder = account,
-                Amount = amount.ToBig()
+                Amount = amount
             };
 
             return gethNode.SendTransaction(tokenAddress, function);

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -5,35 +5,49 @@ using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
 {
-    public partial class Request : RequestBase
+    public interface IHasBlock
+    {
+        BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public byte[] RequestId { get; set; }
 
         public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+
+        [JsonIgnore]
+        public string Id
+        {
+            get
+            {
+                return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant();
+            }
+        }
     }
 
-    public partial class RequestFulfilledEventDTO
+    public partial class RequestFulfilledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO
+    public partial class RequestCancelledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO
+    public partial class SlotFilledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO
+    public partial class SlotFreedEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -1,45 +1,102 @@
-namespace CodexContractsPlugin
+using System.Numerics;
+
+namespace CodexContractsPlugin
 {
     public class TestToken : IComparable<TestToken>
     {
-        public TestToken(decimal amount)
+        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
+
+        public TestToken(BigInteger tstWei)
         {
-            Amount = amount;
+            TstWei = tstWei;
+            Tst = tstWei / WeiFactor;
         }
 
-        public decimal Amount { get; }
+        public BigInteger TstWei { get; }
+        public BigInteger Tst { get; }
 
         public int CompareTo(TestToken? other)
         {
-            return Amount.CompareTo(other!.Amount);
+            return TstWei.CompareTo(other!.TstWei);
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is TestToken token && Amount == token.Amount;
+            return obj is TestToken token && TstWei == token.TstWei;
         }
 
         public override int GetHashCode()
         {
-            return HashCode.Combine(Amount);
+            return HashCode.Combine(TstWei);
         }
 
         public override string ToString()
         {
-            return $"{Amount} TestTokens";
+            var weiOnly = TstWei % WeiFactor;
+
+            var tokens = new List<string>();
+            if (Tst > 0) tokens.Add($"{Tst} TST");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
+
+            return string.Join(" + ", tokens);
+        }
+
+        public static TestToken operator +(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei + b.TstWei);
+        }
+
+        public static bool operator <(TestToken a, TestToken b)
+        {
+            return a.TstWei < b.TstWei;
+        }
+
+        public static bool operator >(TestToken a, TestToken b)
+        {
+            return a.TstWei > b.TstWei;
+        }
+
+        public static bool operator ==(TestToken a, TestToken b)
+        {
+            return a.TstWei == b.TstWei;
+        }
+
+        public static bool operator !=(TestToken a, TestToken b)
+        {
+            return a.TstWei != b.TstWei;
         }
     }
 
-    public static class TokensIntExtensions
+    public static class TestTokensExtensions
     {
-        public static TestToken TestTokens(this int i)
+        public static TestToken TstWei(this int i)
         {
-            return TestTokens(Convert.ToDecimal(i));
+            return TstWei(Convert.ToDecimal(i));
         }
 
-        public static TestToken TestTokens(this decimal i)
+        public static TestToken TstWei(this decimal i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this BigInteger i)
         {
             return new TestToken(i);
         }
+
+        public static TestToken Tst(this int i)
+        {
+            return Tst(Convert.ToDecimal(i));
+        }
+
+        public static TestToken Tst(this decimal i)
+        {
+            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this BigInteger i)
+        {
+            return new TestToken(i * TestToken.WeiFactor);
+        }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,11 +1,13 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Utils;
 
 namespace CodexDiscordBotPlugin
 {
     public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
+        private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!";
         private readonly IPluginTools tools;
 
         public CodexDiscordBotPlugin(IPluginTools tools)
@@ -46,14 +48,59 @@ namespace CodexDiscordBotPlugin
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
+            WaitForStartupMessage(workflow, pod);
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
         }
 
         private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
+            var pod = workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
+            workflow.CreateCrashWatcher(pod.Containers.Single()).Start();
+            return pod;
+        }
+
+        private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod)
+        {
+            var finder = new LogLineFinder(ExpectedStartupMessage, workflow);
+            Time.WaitUntil(() =>
+            {
+                finder.FindLine(pod);
+                return finder.Found;
+            }, nameof(WaitForStartupMessage));
+        }
+
+        public class LogLineFinder : LogHandler
+        {
+            private readonly string message;
+            private readonly IStartupWorkflow workflow;
+
+            public LogLineFinder(string message, IStartupWorkflow workflow)
+            {
+                this.message = message;
+                this.workflow = workflow;
+            }
+
+            public void FindLine(RunningPod pod)
+            {
+                Found = false;
+                foreach (var c in pod.Containers)
+                {
+                    workflow.DownloadContainerLog(c, this);
+                    if (Found) return;
+                }
+            }
+
+            public bool Found { get; private set; }
+
+            protected override void ProcessLine(string line)
+            {
+                if (!Found && line.Contains(message)) Found = true;
+            }
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "codexstorage/codex-discordbot:sha-8c64352";
+        public override string Image => "codexstorage/codex-discordbot:sha-22cf82b";
 
         public static string RewardsPort = "bot_rewards_port";
 
@@ -33,6 +33,8 @@ namespace CodexDiscordBotPlugin
             AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
             AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
+            AddEnvVar("NODISCORD", "1");
+
             AddInternalPortAndVar("REWARDAPIPORT", RewardsPort);
 
             if (!string.IsNullOrEmpty(config.DataPath))

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -27,8 +27,9 @@
 
     public class RewarderBotStartupConfig
     {
-        public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
+        public RewarderBotStartupConfig(string name, string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath)
         {
+            Name = name;
             DiscordBotHost = discordBotHost;
             DiscordBotPort = discordBotPort;
             IntervalMinutes = intervalMinutes;
@@ -37,9 +38,10 @@
             DataPath = dataPath;
         }
 
+        public string Name { get; }
         public string DiscordBotHost { get; }
         public int DiscordBotPort { get; }
-        public string IntervalMinutes { get; }
+        public int IntervalMinutes { get; }
         public DateTime HistoryStartUtc { get; }
         public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,8 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2";
+        public override string Image => "thatbenbierens/codex-rewardbot:newstate";
+            //"codexstorage/codex-rewarderbot:sha-12dc7ef";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -17,7 +18,7 @@ namespace CodexDiscordBotPlugin
 
             AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost);
             AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString());
-            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes);
+            AddEnvVar("INTERVALMINUTES", config.IntervalMinutes.ToString());
             var offset = new DateTimeOffset(config.HistoryStartUtc);
             AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString());
 

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "0F-C8-02-1E-2C-2C-15-F6-91-6A-01-31-11-49-95-06-79-26-25-BF-27-3C-A8-2E-5F-7F-34-FD-C0-57-A0-9A";
+        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -2,25 +2,26 @@
 using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;
 using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
 {
-    public class CodexAccess : ILogHandler
+    public class CodexAccess
     {
+        private readonly ILog log;
         private readonly IPluginTools tools;
         private readonly Mapper mapper = new Mapper();
-        private bool hasContainerCrashed;
 
         public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
         {
             this.tools = tools;
+            log = tools.GetLog();
             Container = container;
             CrashWatcher = crashWatcher;
-            hasContainerCrashed = false;
 
-            CrashWatcher.Start(this);
+            CrashWatcher.Start();
         }
 
         public RunningPod Container { get; }
@@ -34,20 +35,23 @@ namespace CodexPlugin
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
-            var endpoint = GetEndpoint();
-            var str = endpoint.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
+            return CrashCheck(() =>
             {
-                return new DebugPeer
-                {
-                    IsPeerFound = false
-                };
-            }
+                var endpoint = GetEndpoint();
+                var str = endpoint.HttpGetString($"debug/peer/{peerId}");
 
-            var result = endpoint.Deserialize<DebugPeer>(str);
-            result.IsPeerFound = true;
-            return result;
+                if (str.ToLowerInvariant() == "unable to find peer!")
+                {
+                    return new DebugPeer
+                    {
+                        IsPeerFound = false
+                    };
+                }
+
+                var result = endpoint.Deserialize<DebugPeer>(str);
+                result.IsPeerFound = true;
+                return result;
+            });
         }
 
         public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
@@ -59,14 +63,19 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream)
+        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
         {
-            return OnCodex(api => api.UploadAsync(fileStream));
+            return OnCodex(
+                api => api.UploadAsync(fileStream),
+                CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
-        public Stream DownloadFile(string contentId)
+        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
-            var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId));
+            var fileResponse = OnCodex(
+                api => api.DownloadNetworkAsync(contentId),
+                CreateRetryConfig(nameof(DownloadFile), onFailure));
+
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
@@ -89,15 +98,24 @@ namespace CodexPlugin
             return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
+        public CodexSpace Space()
+        {
+            var space = OnCodex<Space>(api => api.SpaceAsync());
+            return mapper.Map(space);
+        }
+
         public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            var endpoint = GetEndpoint();
-            return Time.Retry(() =>
+            return CrashCheck(() =>
             {
-                var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
-                if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-            }, nameof(GetPurchaseStatus));
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+                }, nameof(GetPurchaseStatus));
+            });
 
             // TODO: current getpurchase api does not line up with its openapi spec.
             // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
@@ -114,19 +132,67 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
+        public void LogDiskSpace(string msg)
+        {
+            try
+            {
+                var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
+                Log($"{msg} - Disk info: {diskInfo}");
+            }
+            catch (Exception e)
+            {
+                Log("Failed to get disk info: " + e);
+            }
+        }
+
+        public void DeleteRepoFolder()
+        {
+            try
+            {
+                var containerNumber = Container.Containers.First().Recipe.Number;
+                var dataDir = $"datadir{containerNumber}";
+                var workflow = tools.CreateWorkflow();
+                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
+                Log("Deleted repo folder.");
+            }
+            catch (Exception e)
+            {
+                Log("Unable to delete repo folder: " + e);
+            }
+        }
+
         private T OnCodex<T>(Func<CodexApi, Task<T>> action)
         {
-            var address = GetAddress();
-            var result = tools.CreateHttp(CheckContainerCrashed)
-                .OnClient(client =>
-            {
-                var api = new CodexApi(client);
-                api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
-                return Time.Wait(action(api));
-            });
+            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action));
             return result;
         }
 
+        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
+        {
+            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            return result;
+        }
+
+        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
+        {
+            var address = GetAddress();
+            var api = new CodexApi(client);
+            api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return CrashCheck(() => Time.Wait(action(api)));
+        }
+
+        private T CrashCheck<T>(Func<T> action)
+        {
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                CrashWatcher.HasContainerCrashed();
+            }
+        }
+
         private IEndpoint GetEndpoint()
         {
             return tools
@@ -136,31 +202,67 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
         {
-            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
         }
 
-        public void Log(Stream crashLog)
+        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
         {
-            var log = tools.GetLog();
-            var file = log.CreateSubfile();
-            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
-            file.Write($"Container Crash Log for {Container.Name}.");
+            var timeSet = tools.TimeSet;
 
-            using var reader = new StreamReader(crashLog);
-            var line = reader.ReadLine();
-            while (line != null)
+            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
             {
-                file.Write(line);
-                line = reader.ReadLine();
+                onFailure(failure);
+                Investigate(failure, timeSet);
+            });
+        }
+
+        private void Investigate(Failure failure, ITimeSet timeSet)
+        {
+            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
+                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
+                $"Checking if node responds to debug/info...");
+
+            LogDiskSpace("After retry failure");
+
+            try
+            {
+                var debugInfo = GetDebugInfo();
+                if (string.IsNullOrEmpty(debugInfo.Spr))
+                {
+                    Log("Did not get value debug/info response.");
+                    Throw(failure);
+                }
+                else
+                {
+                    Log("Got valid response from debug/info.");
+                }
+            }
+            catch (Exception ex)
+            {
+                Log("Got exception from debug/info call: " + ex);
+                Throw(failure);
             }
 
-            log.Log("Crash log successfully downloaded.");
-            hasContainerCrashed = true;
+            if (failure.Duration < timeSet.HttpCallTimeout())
+            {
+                Log("Retry failed within HTTP timeout duration.");
+                Throw(failure);
+            }
+        }
+
+        private void Throw(Failure failure)
+        {
+            throw failure.Exception;
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"{GetName()} {msg}");
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-267266a-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
@@ -109,8 +109,9 @@ namespace CodexPlugin
 
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
-                Additional(marketplaceSetup.EthAccount);
+                var account = marketplaceSetup.EthAccountSetup.GetNew();
+                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                Additional(account);
 
                 SetCommandOverride(marketplaceSetup);
                 if (marketplaceSetup.IsValidator)
@@ -119,7 +120,7 @@ namespace CodexPlugin
                 }
             }
 
-            if(!string.IsNullOrEmpty(config.NameOverride))
+            if (!string.IsNullOrEmpty(config.NameOverride))
             {
                 AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
@@ -159,7 +160,7 @@ namespace CodexPlugin
 
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)
         {
-            if (config.StorageQuota != null) return config.StorageQuota;
+            if (config.StorageQuota != null) return config.StorageQuota.Multiply(1.2);
             // Default Codex quota: 8 Gb, using +20% to be safe.
             return 8.GB().Multiply(1.2);
         }

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -15,14 +15,24 @@ namespace CodexPlugin
         DebugInfo GetDebugInfo();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
+        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
+        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
+        CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
         DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
+        EthAccount EthAccount { get; }
+
+        /// <summary>
+        /// Warning! The node is not usable after this.
+        /// TODO: Replace with delete-blocks debug call once available in Codex.
+        /// </summary>
+        void DeleteRepoFolder();
         void Stop(bool waitTillStopped);
     }
 
@@ -30,13 +40,13 @@ namespace CodexPlugin
     {
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
-        private readonly EthAddress? ethAddress;
+        private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
         {
             this.tools = tools;
-            this.ethAddress = ethAddress;
+            this.ethAccount = ethAccount;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
@@ -66,8 +76,17 @@ namespace CodexPlugin
         {
             get
             {
-                if (ethAddress == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
-                return ethAddress;
+                EnsureMarketplace();
+                return ethAccount!.EthAddress;
+            }
+        }
+
+        public EthAccount EthAccount
+        {
+            get
+            {
+                EnsureMarketplace();
+                return ethAccount!;
             }
         }
 
@@ -80,7 +99,7 @@ namespace CodexPlugin
         {
             var debugInfo = CodexAccess.GetDebugInfo();
             var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}");
+            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
             return debugInfo;
         }
 
@@ -91,13 +110,20 @@ namespace CodexPlugin
 
         public ContentId UploadFile(TrackedFile file)
         {
+            return UploadFile(file, DoNothing);
+        }
+
+        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            CodexAccess.LogDiskSpace("Before upload");
+
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
             var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream);
+                return CodexAccess.UploadFile(fileStream, onFailure);
             });
 
             var response = measurement.Value;
@@ -107,15 +133,22 @@ namespace CodexPlugin
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
             Log($"Uploaded file. Received contentId: '{response}'.");
+            CodexAccess.LogDiskSpace("After upload");
+
             return new ContentId(response);
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContent(contentId, DoNothing, fileLabel);
+        }
+
+        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
             transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
@@ -126,6 +159,11 @@ namespace CodexPlugin
             return CodexAccess.LocalFiles();
         }
 
+        public CodexSpace Space()
+        {
+            return CodexAccess.Space();
+        }
+
         public void ConnectToPeer(ICodexNode node)
         {
             var peer = (CodexNode)node;
@@ -142,15 +180,16 @@ namespace CodexPlugin
             return CodexAccess.GetPodInfo();
         }
 
+        public void DeleteRepoFolder()
+        {
+            CodexAccess.DeleteRepoFolder();
+        }
+
         public void Stop(bool waitTillStopped)
         {
+            Log("Stopping...");
             CrashWatcher.Stop();
             Group.Stop(this, waitTillStopped);
-            // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-            //     "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-            //     "available for codex-nodes in groups of 1.");
-            //
-            // Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
@@ -182,12 +221,14 @@ namespace CodexPlugin
                 .ToArray();
         }
 
-        private void DownloadToFile(string contentId, TrackedFile file)
+        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
+            CodexAccess.LogDiskSpace("Before download");
+
             using var fileStream = File.OpenWrite(file.Filename);
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
                 downloadStream.CopyTo(fileStream);
             }
             catch
@@ -195,11 +236,22 @@ namespace CodexPlugin
                 Log($"Failed to download file '{contentId}'.");
                 throw;
             }
+
+            CodexAccess.LogDiskSpace("After download");
+        }
+
+        private void EnsureMarketplace()
+        {
+            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
         }
 
         private void Log(string msg)
         {
             tools.GetLog().Log($"{GetName()}: {msg}");
         }
+
+        private void DoNothing(Failure failure)
+        {
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -22,22 +22,22 @@ namespace CodexPlugin
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
-            var ethAddress = GetEthAddress(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAddress);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAddress);
+            var ethAccount = GetEthAccount(access);
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
+            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAddress? ethAddress)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
         {
-            if (ethAddress == null) return new MarketplaceUnavailable();
+            if (ethAccount == null) return new MarketplaceUnavailable();
             return new MarketplaceAccess(tools.GetLog(), codexAccess);
         }
 
-        private EthAddress? GetEthAddress(CodexAccess access)
+        private EthAccount? GetEthAccount(CodexAccess access)
         {
             var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
             if (ethAccount == null) return null;
-            return ethAccount.EthAddress;
+            return ethAccount;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -168,8 +168,8 @@ namespace CodexPlugin
         public bool IsStorageNode { get; private set; }
         public bool IsValidator { get; private set; }
         public Ether InitialEth { get; private set; } = 0.Eth();
-        public TestToken InitialTestTokens { get; private set; } = 0.TestTokens();
-        public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew();
+        public TestToken InitialTestTokens { get; private set; } = 0.Tst();
+        public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup();
 
         public IMarketplaceSetup AsStorageNode()
         {
@@ -185,7 +185,7 @@ namespace CodexPlugin
 
         public IMarketplaceSetup WithAccount(EthAccount account)
         {
-            EthAccount = account;
+            EthAccountSetup.Pin(account);
             return this;
         }
 
@@ -201,10 +201,41 @@ namespace CodexPlugin
             var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
             result += IsStorageNode ? "(storageNode)" : "()";
             result += IsValidator ? "(validator)" : "() ";
-            result += $"Address: '{EthAccount.EthAddress}' ";
-            result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})";
+            result += $"Address: '{EthAccountSetup}' ";
+            result += $"{InitialEth.Eth} / {InitialTestTokens}";
             result += "] ";
             return result;
         }
     }
+
+    public class EthAccountSetup
+    {
+        private readonly List<EthAccount> accounts = new List<EthAccount>();
+        private bool pinned = false;
+
+        public void Pin(EthAccount account)
+        {
+            accounts.Add(account);
+            pinned = true;
+        }
+
+        public EthAccount GetNew()
+        {
+            if (pinned) return accounts.Last();
+
+            var a = EthAccount.GenerateNew();
+            accounts.Add(a);
+            return a;
+        }
+
+        public EthAccount[] GetAll()
+        {
+            return accounts.ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join(",", accounts.Select(a => a.ToString()).ToArray());
+        }
+    }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -33,7 +33,7 @@ namespace CodexPlugin
             foreach (var rc in containers)
             {
                 var podInfo = GetPodInfo(rc);
-                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
+                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' PodLabel: '{c.RunningPod.StartResult.Deployment.PodLabel}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
                 Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
             }
             LogSeparator();

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -105,4 +105,18 @@ namespace CodexPlugin
             return HashCode.Combine(Id);
         }
     }
+
+    public class CodexSpace
+    {
+        public long TotalBlocks { get; set; }
+        public long QuotaMaxBytes { get; set; }
+        public long QuotaUsedBytes { get; set; }
+        public long QuotaReservedBytes { get; set; }
+        public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+    }
 }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexOpenApi;
 using Newtonsoft.Json.Linq;
 using System.Numerics;
 using Utils;
@@ -62,28 +63,72 @@ namespace CodexPlugin
             };
         }
 
-        public StoragePurchase Map(CodexOpenApi.Purchase purchase)
-        {
-            return new StoragePurchase
-            {
-                State = purchase.State,
-                Error = purchase.Error
-            };
-        }
+        // TODO: Fix openapi spec for this call.
+        //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        //{
+        //    return new StoragePurchase(Map(purchase.Request))
+        //    {
+        //        State = purchase.State,
+        //        Error = purchase.Error
+        //    };
+        //}
+
+        //public StorageRequest Map(CodexOpenApi.StorageRequest request)
+        //{
+        //    return new StorageRequest(Map(request.Ask), Map(request.Content))
+        //    {
+        //        Id = request.Id,
+        //        Client = request.Client,
+        //        Expiry = TimeSpan.FromSeconds(Convert.ToInt64(request.Expiry)),
+        //        Nonce = request.Nonce
+        //    };
+        //}
+
+        //public StorageAsk Map(CodexOpenApi.StorageAsk ask)
+        //{
+        //    return new StorageAsk
+        //    {
+        //        Duration = TimeSpan.FromSeconds(Convert.ToInt64(ask.Duration)),
+        //        MaxSlotLoss = ask.MaxSlotLoss,
+        //        ProofProbability = ask.ProofProbability,
+        //        Reward = Convert.ToDecimal(ask.Reward).TstWei(),
+        //        Slots = ask.Slots,
+        //        SlotSize = new ByteSize(Convert.ToInt64(ask.SlotSize))
+        //    };
+        //}
+
+        //public StorageContent Map(CodexOpenApi.Content content)
+        //{
+        //    return new StorageContent
+        //    {
+        //        Cid = content.Cid
+        //    };
+        //}
 
         public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
         {
             return new StorageAvailability(
-                totalSpace: new Utils.ByteSize(Convert.ToInt64(read.TotalSize)),
+                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
                 maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(Convert.ToDecimal(read.MinPrice)),
-                maxCollateral: new TestToken(Convert.ToDecimal(read.MaxCollateral))
+                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
+                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
             )
             {
                 Id = read.Id
             };
         }
 
+        public CodexSpace Map(Space space)
+        {
+            return new CodexSpace
+            {
+                QuotaMaxBytes = space.QuotaMaxBytes,
+                QuotaReservedBytes = space.QuotaReservedBytes,
+                QuotaUsedBytes = space.QuotaUsedBytes,
+                TotalBlocks = space.TotalBlocks
+            };
+        }
+
         private DebugInfoVersion MapDebugInfoVersion(JObject obj)
         {
             return new DebugInfoVersion
@@ -98,7 +143,7 @@ namespace CodexPlugin
             return new DebugInfoTable
             {
                 LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = new DebugInfoTableNode[0]
+                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
             };
         }
 
@@ -117,6 +162,16 @@ namespace CodexPlugin
             };
         }
 
+        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        {
+            if (nodes == null || nodes.Count == 0)
+            {
+                return new DebugInfoTableNode[0];
+            }
+
+            return nodes.Select(MapDebugInfoTableNode).ToArray();
+        }
+
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
         {
             return new Manifest
@@ -165,8 +220,7 @@ namespace CodexPlugin
 
         private string ToDecInt(TestToken t)
         {
-            var i = new BigInteger(t.Amount);
-            return i.ToString("D");
+            return t.TstWei.ToString("D");
         }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,5 +1,4 @@
 using Logging;
-using Newtonsoft.Json;
 using Utils;
 
 namespace CodexPlugin
@@ -7,7 +6,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
-        StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
+        IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -21,7 +20,7 @@ namespace CodexPlugin
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
 
@@ -38,7 +37,9 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            return new StoragePurchaseContract(log, codexAccess, response, purchase);
+            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase);
+            contract.WaitForStorageContractSubmitted();
+            return contract;
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
@@ -54,7 +55,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Name} {msg}");
+            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
         }
     }
 
@@ -66,7 +67,7 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
-        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
+        public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             Unavailable();
             throw new NotImplementedException();
@@ -78,78 +79,4 @@ namespace CodexPlugin
             throw new InvalidOperationException();
         }
     }
-
-    public class StoragePurchaseContract
-    {
-        private readonly ILog log;
-        private readonly CodexAccess codexAccess;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            Purchase = purchase;
-        }
-
-        public string PurchaseId { get; }
-        public StoragePurchaseRequest Purchase { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            var timeout = Purchase.Expiry + gracePeriod;
-
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.State != lastState)
-                {
-                    lastState = purchaseStatus.State;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    FrameworkAssert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,5 +1,7 @@
 using CodexContractsPlugin;
+using CodexOpenApi;
 using Logging;
+using System.Data;
 using Utils;
 
 namespace CodexPlugin
@@ -12,8 +14,8 @@ namespace CodexPlugin
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TestTokens();
-        public TestToken RequiredCollateral { get; set; } = 1.TestTokens();
+        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
+        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }
@@ -37,6 +39,34 @@ namespace CodexPlugin
     {
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
+        public StorageRequest Request { get; set; } = null!;
+    }
+
+    public class StorageRequest
+    {
+        public string Id { get; set; } = string.Empty;
+        public string Client { get; set; } = string.Empty;
+        public StorageAsk Ask { get; set; } = null!;
+        public StorageContent Content { get; set; } = null!;
+        public string Expiry { get; set; } = string.Empty;
+        public string Nonce { get; set; } = string.Empty;
+    }
+
+    public class StorageAsk
+    {
+        public int Slots { get; set; }
+        public string SlotSize { get; set; } = string.Empty;
+        public string Duration { get; set; } = string.Empty;
+        public string ProofProbability { get; set; } = string.Empty;
+        public string Reward { get; set; } = string.Empty;
+        public int MaxSlotLoss { get; set; }
+    }
+
+    public class StorageContent
+    {
+        public string Cid { get; set; } = string.Empty;
+        //public ErasureParameters Erasure { get; set; }
+        //public PoRParameters Por { get; set; }
     }
 
     public class StorageAvailability

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -0,0 +1,146 @@
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace CodexPlugin
+{
+    public interface IStoragePurchaseContract
+    {
+        string PurchaseId { get; }
+        StoragePurchaseRequest Purchase { get; }
+        ContentId ContentId { get; }
+        void WaitForStorageContractSubmitted();
+        void WaitForStorageContractStarted();
+        void WaitForStorageContractFinished();
+    }
+
+    public class StoragePurchaseContract : IStoragePurchaseContract
+    {
+        private readonly ILog log;
+        private readonly CodexAccess codexAccess;
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly DateTime contractPendingUtc = DateTime.UtcNow;
+        private DateTime? contractSubmittedUtc = DateTime.UtcNow;
+        private DateTime? contractStartedUtc;
+        private DateTime? contractFinishedUtc;
+
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            Purchase = purchase;
+
+            ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
+        }
+
+        public string PurchaseId { get; }
+        public StoragePurchaseRequest Purchase { get; }
+        public ContentId ContentId { get; }
+
+        public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
+        public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
+        public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
+
+        public void WaitForStorageContractSubmitted()
+        {
+            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
+            contractSubmittedUtc = DateTime.UtcNow;
+            LogSubmittedDuration();
+            AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
+        }
+
+        public void WaitForStorageContractStarted()
+        {
+            var timeout = Purchase.Expiry + gracePeriod;
+
+            WaitForStorageContractState(timeout, "started");
+            contractStartedUtc = DateTime.UtcNow;
+            LogStartedDuration();
+            AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+            contractFinishedUtc = DateTime.UtcNow;
+            LogFinishedDuration();
+            AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+        }
+
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+
+            Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.State != lastState)
+                {
+                    lastState = purchaseStatus.State;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(sleep);
+
+                if (lastState == "errored")
+                {
+                    FrameworkAssert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
+                }
+            }
+        }
+
+        private void LogSubmittedDuration()
+        {
+            Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
+                $"( < {Time.FormatDuration(gracePeriod)})");
+        }
+
+        private void LogStartedDuration()
+        {
+            Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " +
+                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
+        }
+
+        private void LogFinishedDuration()
+        {
+            Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " +
+                $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})");
+        }
+
+        private void AssertDuration(TimeSpan? span, TimeSpan max, string message)
+        {
+            if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)");
+            if (span.Value.TotalDays >= max.TotalSeconds)
+            {
+                throw new Exception(nameof(MarketplaceAccess) +
+                    $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " +
+                    message);
+            }
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"[{PurchaseId}] {msg}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -289,6 +289,7 @@ components:
           description: "Root hash of the content"
         originalBytes:
           type: integer
+          format: int64
           description: "Length of original content in bytes"
         blockSize:
           type: integer
@@ -303,14 +304,18 @@ components:
         totalBlocks:
           description: "Number of blocks stored by the node"
           type: integer
+          format: int64
         quotaMaxBytes:
           type: integer
+          format: int64
           description: "Maximum storage space used by the node"
         quotaUsedBytes:
           type: integer
+          format: int64
           description: "Amount of storage space currently in use"
         quotaReservedBytes:
           type: integer
+          format: int64
           description: "Amount of storage space reserved"
 
 servers:

ProjectPlugins/GethPlugin/EthAccount.cs

@@ -24,5 +24,10 @@ namespace GethPlugin
 
             return new EthAccount(ethAddress, account.PrivateKey);
         }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -1,4 +1,5 @@
 using Core;
+using IdentityModel;
 using KubernetesWorkflow.Types;
 using Logging;
 using System.Globalization;
@@ -177,6 +178,41 @@ namespace MetricsPlugin
         {
             return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
         }
+
+        public string AsCsv()
+        {
+            var allTimestamps = Sets.SelectMany(s => s.Values.Select(v => v.Timestamp)).Distinct().OrderDescending().ToArray();
+
+            var lines = new List<string>();
+            MakeLine(lines, e =>
+            {
+                e.Add("Metrics");
+                foreach (var ts in allTimestamps) e.Add(ts.ToEpochTime().ToString());
+            });
+
+            foreach (var set in Sets)
+            {
+                MakeLine(lines, e =>
+                {
+                    e.Add(set.Name);
+                    foreach (var ts in allTimestamps)
+                    {
+                        var value = set.Values.SingleOrDefault(v => v.Timestamp == ts);
+                        if (value == null) e.Add(" ");
+                        else e.Add(value.Value.ToString());
+                    }
+                });
+            }
+
+            return string.Join(Environment.NewLine, lines.ToArray());
+        }
+
+        private void MakeLine(List<string> lines, Action<List<string>> values)
+        {
+            var list = new List<string>();
+            values(list);
+            lines.Add(string.Join(",", list));
+        }
     }
 
     public class MetricsSet

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -148,7 +148,7 @@ namespace ContinuousTests
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
-            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
+            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
         }
 
         private void PerformCleanup(ILog log)
@@ -157,7 +157,7 @@ namespace ContinuousTests
             log.Log("Cleaning up test namespace...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
-            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
+            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
             log.Log("Cleanup finished.");
         }
     }

Tests/CodexContinuousTests/NodeRunner.cs

@@ -64,7 +64,7 @@ namespace ContinuousTests
             }
             finally
             {
-                entryPoint.Tools.CreateWorkflow().DeleteNamespace();
+                entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false);
             }
         }
 

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -54,7 +54,8 @@ namespace ContinuousTests
 
                     entryPoint.Decommission(
                         deleteKubernetesResources: false, // This would delete the continuous test net.
-                        deleteTrackedFiles: true
+                        deleteTrackedFiles: true,
+                        waitTillDone: false
                     );
                     runFinishedHandle.Set();
                 }

Tests/CodexLongTests/BasicTests/LargeFileTests.cs

@@ -50,8 +50,6 @@ namespace CodexLongTests.BasicTests
 
             var node = StartCodex(s => s.WithStorageQuota((size + 10).MB()));
 
-            Time.SetRetryFailedCallback(i => OnFailed(i, node));
-
             var uploadStart = DateTime.UtcNow;
             var cid = node.UploadFile(expectedFile);
             var downloadStart = DateTime.UtcNow;
@@ -62,17 +60,6 @@ namespace CodexLongTests.BasicTests
             AssertTimeConstraint(uploadStart, downloadStart, downloadFinished, size);
         }
 
-        private void OnFailed(int tries, ICodexNode node)
-        {
-            if (tries < 5) return;
-
-            if (tries % 10 == 0)
-            {
-                Log($"After try {tries}, downloading node log.");
-                Ci.DownloadLog(node);
-            }
-        }
-
         private void AssertTimeConstraint(DateTime uploadStart, DateTime downloadStart, DateTime downloadFinished, long size)
         {
             float sizeInMB = size;

Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -15,9 +15,9 @@ namespace CodexLongTests.DownloadConnectivityTests
             [Values(10, 15, 20)] int numberOfNodes,
             [Values(10, 100)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) StartCodex();
+            var nodes = StartCodex(numberOfNodes);
 
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
         }
     }
 }

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -17,12 +17,13 @@ public class ScalabilityTests : CodexDistTest
     [Combinatorial]
     [UseLongTimeouts]
     [DontDownloadLogs]
+    [WaitForCleanup]
     public void ShouldMaintainFileInNetwork(
-        [Values(10, 40, 80, 100)] int numberOfNodes,
-        [Values(100, 1000, 5000, 10000)] int fileSizeInMb
+        [Values(4, 5, 6)] int numberOfNodes, // TODO: include 10, 40, 80 and 100, not 5
+        [Values(4000, 5000, 6000, 7000, 8000, 9000, 10000)] int fileSizeInMb
     )
     {
-        var logLevel = CodexLogLevel.Info;
+        var logLevel = CodexLogLevel.Trace;
 
         var bootstrap = StartCodex(s => s.WithLogLevel(logLevel));
         var nodes = StartCodex(numberOfNodes - 1, s => s
@@ -35,17 +36,27 @@ public class ScalabilityTests : CodexDistTest
         var downloader = nodes.PickOneRandom();
 
         var testFile = GenerateTestFile(fileSizeInMb.MB());
-        var contentId = uploader.UploadFile(testFile);
-        var downloadedFile = downloader.DownloadContent(contentId);
+
+        LogNodeStatus(uploader);
+        var contentId = uploader.UploadFile(testFile, f => LogNodeStatus(uploader));
+        LogNodeStatus(uploader);
+
+        LogNodeStatus(downloader);
+        var downloadedFile = downloader.DownloadContent(contentId, f => LogNodeStatus(downloader));
+        LogNodeStatus(downloader);
 
         downloadedFile!.AssertIsEqual(testFile);
 
+        uploader.DeleteRepoFolder();
         uploader.Stop(true);
 
         var otherDownloader = nodes.PickOneRandom();
         downloadedFile = otherDownloader.DownloadContent(contentId);
 
         downloadedFile!.AssertIsEqual(testFile);
+
+        downloader.DeleteRepoFolder();
+        otherDownloader.DeleteRepoFolder();
     }
 
     /// <summary>
@@ -57,6 +68,7 @@ public class ScalabilityTests : CodexDistTest
     [Combinatorial]
     [UseLongTimeouts]
     [DontDownloadLogs]
+    [WaitForCleanup]
     public void EveryoneGetsAFile(
         [Values(10, 40, 80, 100)] int numberOfNodes,
         [Values(100, 1000, 5000, 10000)] int fileSizeInMb

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -45,6 +45,9 @@ namespace CodexTests.BasicTests
 
             metrics[0].AssertThat("libp2p_peers", Is.EqualTo(1));
             metrics[1].AssertThat("libp2p_peers", Is.EqualTo(1));
+
+            LogNodeStatus(primary, metrics[0]);
+            LogNodeStatus(primary2, metrics[1]);
         }
 
         [Test]

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -13,8 +13,8 @@ namespace CodexTests.BasicTests
         [Test]
         public void MarketplaceExample()
         {
-            var hostInitialBalance = 234.TestTokens();
-            var clientInitialBalance = 100000.TestTokens();
+            var hostInitialBalance = 234.TstWei();
+            var clientInitialBalance = 100000.TstWei();
             var fileSize = 10.MB();
 
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
@@ -33,7 +33,7 @@ namespace CodexTests.BasicTests
                     .AsStorageNode()
                     .AsValidator()));
 
-            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
+            var expectedHostBalance = (numberOfHosts * hostInitialBalance.TstWei).TstWei();
             foreach (var host in hosts)
             {
                 AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
@@ -41,8 +41,8 @@ namespace CodexTests.BasicTests
                 var availability = new StorageAvailability(
                     totalSpace: 10.GB(),
                     maxDuration: TimeSpan.FromMinutes(30),
-                    minPriceForTotalSpace: 1.TestTokens(),
-                    maxCollateral: 20.TestTokens()
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 20.TstWei()
                 );
                 host.Marketplace.MakeStorageAvailable(availability);
             }
@@ -60,13 +60,13 @@ namespace CodexTests.BasicTests
 
             var purchase = new StoragePurchaseRequest(contentId)
             {
-                PricePerSlotPerSecond = 2.TestTokens(),
-                RequiredCollateral = 10.TestTokens(),
+                PricePerSlotPerSecond = 2.TstWei(),
+                RequiredCollateral = 10.TstWei(),
                 MinRequiredNumberOfNodes = 5,
                 NodeFailureTolerance = 2,
                 ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
+                Duration = TimeSpan.FromMinutes(6),
+                Expiry = TimeSpan.FromMinutes(5)
             };
 
             var purchaseContract = client.Marketplace.RequestStorage(purchase);
@@ -85,16 +85,56 @@ namespace CodexTests.BasicTests
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
         }
 
+        [Test]
+        public void CanDownloadContentFromContractCid()
+        {
+            var fileSize = 10.MB();
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var contracts = Ci.StartCodexContracts(geth);
+            var testFile = GenerateTestFile(fileSize);
+
+            var client = StartCodex(s => s
+                .WithName("Client")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), 10.Tst())));
+
+            var uploadCid = client.UploadFile(testFile);
+
+            var purchase = new StoragePurchaseRequest(uploadCid)
+            {
+                PricePerSlotPerSecond = 2.TstWei(),
+                RequiredCollateral = 10.TstWei(),
+                MinRequiredNumberOfNodes = 5,
+                NodeFailureTolerance = 2,
+                ProofProbability = 5,
+                Duration = TimeSpan.FromMinutes(5),
+                Expiry = TimeSpan.FromMinutes(4)
+            };
+
+            var purchaseContract = client.Marketplace.RequestStorage(purchase);
+            var contractCid = purchaseContract.ContentId;
+            Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
+
+            // Download both from client.
+            testFile.AssertIsEqual(client.DownloadContent(uploadCid));
+            testFile.AssertIsEqual(client.DownloadContent(contractCid));
+
+            // Download both from another node.
+            var downloader = StartCodex(s => s.WithName("Downloader"));
+            testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
+            testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
+        }
+
         private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)
         {
             Time.Retry(() =>
             {
-                var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange());
-                var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
+                var events = contracts.GetEvents(GetTestRunTimeRange());
+                var slotFilledEvents = events.GetSlotFilledEvents();
 
-                Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
-
-                if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
+                var msg = $"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}";
+                Debug(msg);
+                if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception(msg);
             }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
         }
 
@@ -107,7 +147,8 @@ namespace CodexTests.BasicTests
 
         private Request GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth)
         {
-            var requests = contracts.GetStorageRequests(geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()));
+            var events = contracts.GetEvents(GetTestRunTimeRange());
+            var requests = events.GetStorageRequests();
             Assert.That(requests.Length, Is.EqualTo(1));
             return requests.Single();
         }

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -13,18 +13,8 @@ namespace CodexTests.BasicTests
             var primary = StartCodex();
 
             PerformOneClientTest(primary);
-        }
 
-        [Test]
-        public void RestartTest()
-        {
-            var primary = StartCodex();
-
-            primary.Stop(waitTillStopped: true);
-
-            primary = StartCodex();
-
-            PerformOneClientTest(primary);
+            LogNodeStatus(primary);
         }
 
         private void PerformOneClientTest(ICodexNode primary)

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using CodexPlugin;
+using NUnit.Framework;
 using Utils;
 
 namespace CodexTests.BasicTests
@@ -20,5 +21,34 @@ namespace CodexTests.BasicTests
 
             testFile.AssertIsEqual(downloadedFile);
         }
+
+        [Test]
+        public void DownloadingUnknownCidDoesNotCauseCrash()
+        {
+            var node = StartCodex(2).First();
+
+            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
+
+            try
+            {
+                node.DownloadContent(unknownCid);
+            }
+            catch (Exception ex)
+            {
+                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
+                {
+                    throw;
+                }
+            }
+
+            // Check that the node stays alive for at least another 5 minutes.
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
+            {
+                Thread.Sleep(5000);
+                var info = node.GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(info.Id));
+            }
+        }
     }
 }

Tests/CodexTests/CodexDistTest.cs

@@ -6,14 +6,14 @@ using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
 using DistTestCore.Logs;
+using MetricsPlugin;
+using Newtonsoft.Json;
 using NUnit.Framework.Constraints;
 
 namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {
-        private readonly Dictionary<TestLifecycle, List<ICodexNode>> onlineCodexNodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
-
         public CodexDistTest()
         {
             ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
@@ -29,16 +29,6 @@ namespace CodexTests
             localBuilder.Build();
         }
 
-        protected override void LifecycleStart(TestLifecycle lifecycle)
-        {
-            onlineCodexNodes.Add(lifecycle, new List<ICodexNode>());
-        }
-
-        protected override void LifecycleStop(TestLifecycle lifecycle)
-        {
-            onlineCodexNodes.Remove(lifecycle);
-        }
-
         public ICodexNode StartCodex()
         {
             return StartCodex(s => { });
@@ -61,7 +51,7 @@ namespace CodexTests
                 setup(s);
                 OnCodexSetup(s);
             });
-            onlineCodexNodes[Get()].AddRange(group);
+
             return group;
         }
 
@@ -75,11 +65,6 @@ namespace CodexTests
             return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager());
         }
 
-        public IEnumerable<ICodexNode> GetAllOnlineCodexNodes()
-        {
-            return onlineCodexNodes[Get()];
-        }
-
         public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
         {
             AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
@@ -99,17 +84,29 @@ namespace CodexTests
             log.AssertLogDoesNotContain("ERR ");
         }
 
+        public void LogNodeStatus(ICodexNode node, IMetricsAccess? metrics = null)
+        {
+            Log("Status for " + node.GetName() + Environment.NewLine +
+                GetBasicNodeStatus(node));
+        }
+
+        private string GetBasicNodeStatus(ICodexNode node)
+        {
+            return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine +
+                node.Space().ToString() + Environment.NewLine;
+        }
+
+        // Disabled for now: Makes huge log files!
+        //private string GetNodeMetrics(IMetricsAccess? metrics)
+        //{
+        //    if (metrics == null) return "No metrics enabled";
+        //    var m = metrics.GetAllMetrics();
+        //    if (m == null) return "No metrics received";
+        //    return m.AsCsv();
+        //}
+
         protected virtual void OnCodexSetup(ICodexSetup setup)
         {
         }
-
-        protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
-        {
-            var nodes = onlineCodexNodes[lifecycle];
-            var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
-            var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
-            if (upload != null) data.Add("avgupload", upload.ToString());
-            if (download != null) data.Add("avgdownload", download.ToString());
-        }
     }
 }

Tests/CodexTests/CodexTests.csproj

@@ -13,11 +13,13 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\MetricsPlugin\MetricsPlugin.csproj" />
+    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexPlugin;
 using GethPlugin;
 using NUnit.Framework;
 using Utils;
@@ -11,9 +12,9 @@ namespace CodexTests.DownloadConnectivityTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDownload()
         {
-            StartCodex(2, s => s.EnableMetrics());
+            var nodes = StartCodex(2, s => s.EnableMetrics());
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes);
         }
 
         [Test]
@@ -21,10 +22,10 @@ namespace CodexTests.DownloadConnectivityTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
-                .WithInitial(10.Eth(), 1000.TestTokens())));
+            var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+                .WithInitial(10.Eth(), 1000.TstWei())));
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes);
         }
 
         [Test]
@@ -33,14 +34,14 @@ namespace CodexTests.DownloadConnectivityTests
             [Values(2, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            StartCodex(numberOfNodes);
+            var nodes = StartCodex(numberOfNodes);
 
-            AssertAllNodesConnected(sizeMBs);
+            AssertAllNodesConnected(nodes, sizeMBs);
         }
 
-        private void AssertAllNodesConnected(int sizeMBs = 10)
+        private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes, int sizeMBs = 10)
         {
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB());
         }
     }
 }

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using CodexPlugin;
+using NUnit.Framework;
 
 namespace CodexTests.PeerDiscoveryTests
 {
@@ -13,7 +14,7 @@ namespace CodexTests.PeerDiscoveryTests
             var l1Node = StartCodex(s => s.WithBootstrapNode(root));
             var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node));
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(root, l1Source, l1Node, l2Target);
         }
 
         [Test]
@@ -25,7 +26,7 @@ namespace CodexTests.PeerDiscoveryTests
             var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node));
             var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node));
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(root, l1Source, l1Node, l2Node, l3Target);
         }
 
         [TestCase(3)]
@@ -33,18 +34,22 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void NodeChainTest(int chainLength)
         {
+            var nodes = new List<ICodexNode>();
             var node = StartCodex();
+            nodes.Add(node);    
+
             for (var i = 1; i < chainLength; i++)
             {
                 node = StartCodex(s => s.WithBootstrapNode(node));
+                nodes.Add(node);
             }
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes.ToArray());
         }
 
-        private void AssertAllNodesConnected()
+        private void AssertAllNodesConnected(params ICodexNode[] nodes)
         {
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
         }
     }
 }

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -21,9 +21,9 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void MetricsDoesNotInterfereWithPeerDiscovery()
         {
-            StartCodex(2, s => s.EnableMetrics());
+            var nodes = StartCodex(2, s => s.EnableMetrics());
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes);
         }
 
         [Test]
@@ -31,10 +31,10 @@ namespace CodexTests.PeerDiscoveryTests
         {
             var geth = Ci.StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
-            StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
-                .WithInitial(10.Eth(), 1000.TestTokens())));
+            var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
+                .WithInitial(10.Eth(), 1000.TstWei())));
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes);
         }
 
         [TestCase(2)]
@@ -42,16 +42,15 @@ namespace CodexTests.PeerDiscoveryTests
         [TestCase(10)]
         public void VariableNodes(int number)
         {
-            StartCodex(number);
+            var nodes = StartCodex(number);
 
-            AssertAllNodesConnected();
+            AssertAllNodesConnected(nodes);
         }
 
-        private void AssertAllNodesConnected()
+        private void AssertAllNodesConnected(IEnumerable<ICodexNode> nodes)
         {
-            var allNodes = GetAllOnlineCodexNodes();
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes);
-            CheckRoutingTable(allNodes);
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes);
+            CheckRoutingTable(nodes);
         }
 
         private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes)

Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs

@@ -3,7 +3,7 @@ using Logging;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.ScalabilityTests
+namespace CodexTests.UtilityTests
 {
     [TestFixture]
     public class ClusterDiscSpeedTests : DistTest
@@ -18,7 +18,7 @@ namespace CodexTests.ScalabilityTests
         )
         {
             long targetSize = (long)(1024 * 1024 * 1024) * 2;
-            long bufferSizeBytes = ((long)bufferSizeKb) * 1024;
+            long bufferSizeBytes = (long)bufferSizeKb * 1024;
 
             var filename = nameof(DiscSpeedTest);
 
@@ -28,7 +28,7 @@ namespace CodexTests.ScalabilityTests
             var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename);
             Thread.Sleep(2000);
             var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename);
-            
+
             Log($"Write speed: {writeSpeed} per second.");
             Log($"Read speed: {readSpeed} per second.");
         }

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -1,7 +1,13 @@
 using CodexContractsPlugin;
 using CodexDiscordBotPlugin;
 using CodexPlugin;
+using Core;
+using DiscordRewards;
+using DistTestCore;
 using GethPlugin;
+using KubernetesWorkflow.Types;
+using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using Utils;
 
@@ -10,21 +16,155 @@ namespace CodexTests.UtilityTests
     [TestFixture]
     public class DiscordBotTests : AutoBootstrapDistTest
     {
+        private readonly RewardRepo repo = new RewardRepo();
+        private readonly TestToken hostInitialBalance = 3000000.TstWei();
+        private readonly TestToken clientInitialBalance = 1000000000.TstWei();
+        private readonly EthAccount clientAccount = EthAccount.GenerateNew();
+        private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
+        private readonly List<ulong> rewardsSeen = new List<ulong>();
+        private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
+        private readonly List<string> receivedEvents = new List<string>();
+        private readonly List<MarketAverage> receivedAverages = new List<MarketAverage>();
+ 
         [Test]
-        [Ignore("Used for debugging bots")]
+        [DontDownloadLogs]
         public void BotRewardTest()
         {
-            var myAccount = EthAccount.GenerateNew();
-
-            var sellerInitialBalance = 234.TestTokens();
-            var buyerInitialBalance = 100000.TestTokens();
-            var fileSize = 11.MB();
-
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
+            var gethInfo = CreateGethInfo(geth, contracts);
 
-            // start bot and rewarder
-            var gethInfo = new DiscordBotGethInfo(
+            var botContainer = StartDiscordBot(gethInfo);
+            var rewarderContainer = StartRewarderBot(gethInfo, botContainer);
+
+            StartHosts(geth, contracts);
+            var client = StartClient(geth, contracts);
+
+            var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer);
+            apiCalls.Start(OnCommand);
+
+            var purchaseContract = ClientPurchasesStorage(client);
+            purchaseContract.WaitForStorageContractStarted();
+            purchaseContract.WaitForStorageContractFinished();
+            Thread.Sleep(rewarderInterval * 3);
+
+            apiCalls.Stop();
+
+            AssertEventOccurance("Created as New.", 1);
+            AssertEventOccurance("SlotFilled", Convert.ToInt32(GetNumberOfRequiredHosts()));
+            AssertEventOccurance("Transit: New -> Started", 1);
+            AssertEventOccurance("Transit: Started -> Finished", 1);
+
+            AssertMarketAverage();
+
+            foreach (var r in repo.Rewards)
+            {
+                var seen = rewardsSeen.Any(s => r.RoleId == s);
+
+                Log($"{Lookup(r.RoleId)} = {seen}");
+            }
+
+            Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId)));
+        }
+
+        private string Lookup(ulong rewardId)
+        {
+            var reward = repo.Rewards.Single(r => r.RoleId == rewardId);
+            return $"({rewardId})'{reward.Message}'";
+        }
+
+        private void AssertEventOccurance(string msg, int expectedCount)
+        {
+            Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
+                $"Event '{msg}' did not occure correct number of times.");
+        }
+
+        private void AssertMarketAverage()
+        {
+            Assert.That(receivedAverages.Count, Is.EqualTo(1));
+            var a = receivedAverages.Single();
+
+            Assert.That(a.NumberOfFinished, Is.EqualTo(1));
+            Assert.That(a.TimeRangeSeconds, Is.EqualTo(5760));
+            Assert.That(a.Price, Is.EqualTo(2.0f).Within(0.1f));
+            Assert.That(a.Size, Is.EqualTo(GetMinFileSize().SizeInBytes).Within(1.0f));
+            Assert.That(a.Duration, Is.EqualTo(GetMinRequiredRequestDuration().TotalSeconds).Within(1.0f));
+            Assert.That(a.Collateral, Is.EqualTo(10.0f).Within(0.1f));
+            Assert.That(a.ProofProbability, Is.EqualTo(5.0f).Within(0.1f));
+        }
+
+        private void OnCommand(string timestamp, GiveRewardsCommand call)
+        {
+            Log($"<API call {timestamp}>");
+            receivedAverages.AddRange(call.Averages);
+            foreach (var a in call.Averages)
+            {
+                Log("\tAverage: " + JsonConvert.SerializeObject(a));
+            }
+            receivedEvents.AddRange(call.EventsOverview);
+            foreach (var e in call.EventsOverview)
+            {
+                Log("\tEvent: " + e);
+            }
+            foreach (var r in call.Rewards)
+            {
+                var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId);
+                if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId);
+                foreach (var address in r.UserAddresses)
+                {
+                    var user = IdentifyAccount(address);
+                    Log("\tReward: " + user + ": " + reward.Message);
+                }
+            }
+            Log($"</API call>");
+        }
+
+        private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client)
+        {
+            var testFile = GenerateTestFile(GetMinFileSize());
+            var contentId = client.UploadFile(testFile);
+            var purchase = new StoragePurchaseRequest(contentId)
+            {
+                PricePerSlotPerSecond = 2.TstWei(),
+                RequiredCollateral = 10.TstWei(),
+                MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(),
+                NodeFailureTolerance = 2,
+                ProofProbability = 5,
+                Duration = GetMinRequiredRequestDuration(),
+                Expiry = GetMinRequiredRequestDuration() - TimeSpan.FromMinutes(1)
+            };
+
+            return client.Marketplace.RequestStorage(purchase);
+        }
+
+        private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts)
+        {
+            var node = StartCodex(s => s
+                .WithName("Client")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithAccount(clientAccount)
+                    .WithInitial(10.Eth(), clientInitialBalance)));
+
+            Log($"Client {node.EthAccount.EthAddress}");
+            return node;
+        }
+
+        private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer)
+        {
+            return Ci.DeployRewarderBot(new RewarderBotStartupConfig(
+                name: "rewarder-bot",
+                discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host,
+                discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port,
+                intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)),
+                historyStartUtc: DateTime.UtcNow,
+                gethInfo: gethInfo,
+                dataPath: null
+            ));
+        }
+
+        private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts)
+        {
+            return new DiscordBotGethInfo(
                 host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host,
                 port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port,
                 privKey: geth.StartResult.Account.PrivateKey,
@@ -32,8 +172,12 @@ namespace CodexTests.UtilityTests
                 tokenAddress: contracts.Deployment.TokenAddress,
                 abi: contracts.Deployment.Abi
             );
+        }
+
+        private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo)
+        {
             var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
-                name: "bot",
+                name: "discord-bot",
                 token: "aaa",
                 serverName: "ThatBen's server",
                 adminRoleName: "bottest-admins",
@@ -42,70 +186,190 @@ namespace CodexTests.UtilityTests
                 kubeNamespace: "notneeded",
                 gethInfo: gethInfo
             ));
-            var botContainer = bot.Containers.Single();
-            Ci.DeployRewarderBot(new RewarderBotStartupConfig(
-                //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host,
-                //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port,
-                discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host,
-                discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port,
-                intervalMinutes: "1",
-                historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3),
-                gethInfo: gethInfo,
-                dataPath: null
-            ));
+            return bot.Containers.Single();
+        }
 
-            var numberOfHosts = 3;
+        private void StartHosts(IGethNode geth, ICodexContracts contracts)
+        {
+            var hosts = StartCodex(GetNumberOfLiveHosts(), s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                {
+                    ContractClock = CodexLogLevel.Trace,
+                })
+                .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts()))
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
 
-            for (var i = 0; i < numberOfHosts; i++)
+            var availability = new StorageAvailability(
+                totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()),
+                maxDuration: TimeSpan.FromMinutes(30),
+                minPriceForTotalSpace: 1.TstWei(),
+                maxCollateral: hostInitialBalance
+            );
+
+            foreach (var host in hosts)
             {
-                var seller = StartCodex(s => s
-                    .WithName("Seller")
-                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                    {
-                        ContractClock = CodexLogLevel.Trace,
-                    })
-                    .WithStorageQuota(11.GB())
-                    .EnableMarketplace(geth, contracts, m => m
-                        .WithAccount(myAccount)
-                        .WithInitial(10.Eth(), sellerInitialBalance)
-                        .AsStorageNode()
-                        .AsValidator()));
+                hostAccounts.Add(host.EthAccount);
+                host.Marketplace.MakeStorageAvailable(availability);
+            }
+        }
 
-                var availability = new StorageAvailability(
-                    totalSpace: 10.GB(),
-                    maxDuration: TimeSpan.FromMinutes(30),
-                    minPriceForTotalSpace: 1.TestTokens(),
-                    maxCollateral: 20.TestTokens()
-                );
-                seller.Marketplace.MakeStorageAvailable(availability);
+        private int GetNumberOfLiveHosts()
+        {
+            return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3;
+        }
+
+        private ByteSize Mult(ByteSize size, int mult)
+        {
+            return new ByteSize(size.SizeInBytes * mult);
+        }
+
+        private ByteSize GetMinFileSizePlus(int plusMb)
+        {
+            return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes);
+        }
+
+        private ByteSize GetMinFileSize()
+        {
+            ulong minSlotSize = 0;
+            ulong minNumHosts = 0;
+            foreach (var r in repo.Rewards)
+            {
+                var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes);
+                var h = r.CheckConfig.MinNumberOfHosts;
+                if (s > minSlotSize) minSlotSize = s;
+                if (h > minNumHosts) minNumHosts = h;
             }
 
-            var testFile = GenerateTestFile(fileSize);
+            var minFileSize = ((minSlotSize + 1024) * minNumHosts);
+            return new ByteSize(Convert.ToInt64(minFileSize));
+        }
 
-            var buyer = StartCodex(s => s
-                .WithName("Buyer")
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithAccount(myAccount)
-                    .WithInitial(10.Eth(), buyerInitialBalance)));
+        private uint GetNumberOfRequiredHosts()
+        {
+            return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts));
+        }
 
-            var contentId = buyer.UploadFile(testFile);
+        private TimeSpan GetMinRequiredRequestDuration()
+        {
+            return repo.Rewards.Max(r => r.CheckConfig.MinDuration) + TimeSpan.FromSeconds(10);
+        }
 
-            var purchase = new StoragePurchaseRequest(contentId)
+        private string IdentifyAccount(string address)
+        {
+            if (address == clientAccount.EthAddress.Address) return "Client";
+            try
             {
-                PricePerSlotPerSecond = 2.TestTokens(),
-                RequiredCollateral = 10.TestTokens(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(6),
-                Expiry = TimeSpan.FromMinutes(5)
-            };
+                var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address);
+                return "Host" + index;
+            }
+            catch
+            {
+                return "UNKNOWN";
+            }
+        }
 
-            var purchaseContract = buyer.Marketplace.RequestStorage(purchase);
+        public class RewardApiCalls
+        {
+            private readonly ContainerFileMonitor monitor;
 
-            purchaseContract.WaitForStorageContractStarted();
+            public RewardApiCalls(ILog log, CoreInterface ci, RunningContainer botContainer)
+            {
+                monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log");
+            }
 
-            purchaseContract.WaitForStorageContractFinished();
+            public void Start(Action<string, GiveRewardsCommand> onCommand)
+            {
+                monitor.Start(line => ParseLine(line, onCommand));
+            }
+
+            public void Stop()
+            {
+                monitor.Stop();
+            }
+
+            private void ParseLine(string line, Action<string, GiveRewardsCommand> onCommand)
+            {
+                try
+                {
+                    var timestamp = line.Substring(0, 30);
+                    var json = line.Substring(31);
+
+                    var cmd = JsonConvert.DeserializeObject<GiveRewardsCommand>(json);
+                    if (cmd != null)
+                    {
+                        onCommand(timestamp, cmd);
+                    }
+                }
+                catch
+                {
+                }
+            }
+        }
+
+        public class ContainerFileMonitor
+        {
+            private readonly ILog log;
+            private readonly CoreInterface ci;
+            private readonly RunningContainer botContainer;
+            private readonly string filePath;
+            private readonly CancellationTokenSource cts = new CancellationTokenSource();
+            private readonly List<string> seenLines = new List<string>();
+            private Task worker = Task.CompletedTask;
+            private Action<string> onNewLine = c => { };
+
+            public ContainerFileMonitor(ILog log, CoreInterface ci, RunningContainer botContainer, string filePath)
+            {
+                this.log = log;
+                this.ci = ci;
+                this.botContainer = botContainer;
+                this.filePath = filePath;
+            }
+
+            public void Start(Action<string> onNewLine)
+            {
+                this.onNewLine = onNewLine;
+                worker = Task.Run(Worker);
+            }
+
+            public void Stop()
+            {
+                cts.Cancel();
+                worker.Wait();
+            }
+
+            // did any container crash? that's why it repeats?
+
+
+            private void Worker()
+            {
+                while (!cts.IsCancellationRequested)
+                {
+                    Update();
+                }
+            }
+
+            private void Update()
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(10));
+                if (cts.IsCancellationRequested) return;
+
+                var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath);
+                var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
+                foreach (var line in lines)
+                {
+                    // log.Log("line: " + line);
+
+                    if (!seenLines.Contains(line))
+                    {
+                        seenLines.Add(line);
+                        onNewLine(line);
+                    }
+                }
+            }
         }
     }
 }

Tests/DistTestCore/DistTest.cs

@@ -52,7 +52,7 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
+                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
                 });                
             }
             catch (Exception ex)
@@ -72,7 +72,8 @@ namespace DistTestCore
             globalEntryPoint.Decommission(
                 // There shouldn't be any of either, but clean everything up regardless.
                 deleteKubernetesResources: true,
-                deleteTrackedFiles: true
+                deleteTrackedFiles: true,
+                waitTillDone: true
             );
         }
 
@@ -185,7 +186,13 @@ namespace DistTestCore
                 lock (lifecycleLock)
                 {
                     var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
+                    var lifecycle = new TestLifecycle(
+                        fixtureLog.CreateTestLog(),
+                        configuration,
+                        GetTimeSet(),
+                        testNamespace,
+                        deployId,
+                        ShouldWaitForCleanup());
                     lifecycles.Add(testName, lifecycle);
                     LifecycleStart(lifecycle);
                 }
@@ -208,7 +215,7 @@ namespace DistTestCore
                 IncludeLogsOnTestFailure(lifecycle);
                 LifecycleStop(lifecycle);
                 lifecycle.DeleteAllResources();
-                lifecycle = null!;
+                lifecycles.Remove(GetCurrentTestName());
             });
         }
 
@@ -235,6 +242,11 @@ namespace DistTestCore
             return new DefaultTimeSet();
         }
 
+        private bool ShouldWaitForCleanup()
+        {
+            return CurrentTestMethodHasAttribute<WaitForCleanupAttribute>();
+        }
+
         private bool ShouldUseLongTimeouts()
         {
             return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();

Tests/DistTestCore/TestLifecycle.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
         {
             Log = log;
             Configuration = configuration;
@@ -27,7 +27,7 @@ namespace DistTestCore
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
             this.deployId = deployId;
-
+            WaitForCleanup = waitForCleanup;
             log.WriteLogTag();
         }
 
@@ -35,13 +35,15 @@ namespace DistTestCore
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
 
         public void DeleteAllResources()
         {
             entryPoint.Decommission(
                 deleteKubernetesResources: true,
-                deleteTrackedFiles: true
+                deleteTrackedFiles: true,
+                waitTillDone: WaitForCleanup
             );
         }
 

Tests/DistTestCore/WaitForCleanupAttribute.cs

@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    /// <summary>
+    /// By default, test system does not wait until all resources are destroyed before starting the
+    /// next test. This saves a lot of time but it's not always what you want.
+    /// If you want to be sure the resources of your test are destroyed before the next test starts,
+    /// add this attribute to your test method.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class WaitForCleanupAttribute : PropertyAttribute
+    {
+    }
+}

Tests/FrameworkTests/CodexContractsPlugin/TestTokenTests.cs

@@ -0,0 +1,46 @@
+using CodexContractsPlugin;
+using NUnit.Framework;
+using System.Numerics;
+
+namespace FrameworkTests.CodexContractsPlugin
+{
+    [TestFixture]
+    public class TestTokenTests
+    {
+        private const decimal factor = 1000000000000000000m;
+
+        [Test]
+        public void RepresentsSmallAmount()
+        {
+            var t = 10.TstWei();
+
+            Assert.That(t.TstWei, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(0)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TSTWEI"));
+        }
+
+        [Test]
+        public void RepresentsLargeAmount()
+        {
+            var t = 10.Tst();
+
+            var expected = new BigInteger(10 * factor);
+            Assert.That(t.TstWei, Is.EqualTo(expected));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TST"));
+        }
+
+        [Test]
+        public void RepresentsLongAmount()
+        {
+            var a = 10.Tst();
+            var b = 20.TstWei();
+            var t = a + b;
+
+            var expected = new BigInteger((10 * factor) + 20);
+            Assert.That(t.TstWei, Is.EqualTo(expected));
+            Assert.That(t.Tst, Is.EqualTo(new BigInteger(10)));
+            Assert.That(t.ToString(), Is.EqualTo("10 TST + 20 TSTWEI"));
+        }
+    }
+}

Tests/FrameworkTests/FrameworkTests.csproj

@@ -16,6 +16,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\NethereumWorkflow\NethereumWorkflow.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -32,7 +32,7 @@ namespace BiblioTech.Commands
             }
 
             var eth = 0.Eth();
-            var testTokens = 0.TestTokens();
+            var testTokens = 0.TstWei();
 
             await Task.Run(() =>
             {

Tools/BiblioTech/Commands/MintCommand.cs

@@ -51,7 +51,7 @@ namespace BiblioTech.Commands
         {
             if (ShouldMintTestTokens(contracts, addr))
             {
-                var tokens = Program.Config.MintTT.TestTokens();
+                var tokens = Program.Config.MintTT.TstWei();
                 var transaction = contracts.MintTestTokens(addr, tokens);
                 report.Add($"Minted {tokens} {FormatTransactionLink(transaction)}");
                 return new Transaction<TestToken>(tokens, transaction);
@@ -77,7 +77,7 @@ namespace BiblioTech.Commands
         private bool ShouldMintTestTokens(ICodexContracts contracts, EthAddress addr)
         {
             var testTokens = contracts.GetTestTokenBalance(addr);
-            return testTokens.Amount < Program.Config.MintTT;
+            return testTokens < Program.Config.MintTT.TstWei();
         }
 
         private bool ShouldSendEth(IGethNode gethNode, EthAddress addr)

Tools/BiblioTech/Configuration.cs

@@ -1,4 +1,5 @@
 using ArgsUniform;
+using System.Numerics;
 
 namespace BiblioTech
 {
@@ -31,31 +32,15 @@ namespace BiblioTech
         [Uniform("send-eth", "se", "SENDETH", true, "Amount of Eth send by the mint command.")]
         public int SendEth { get; set; } = 10;
 
-        [Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TestTokens minted by the mint command.")]
-        public int MintTT { get; set; } = 1073741824;
+        [Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TSTWEI minted by the mint command.")]
+        public BigInteger MintTT { get; set; } = 1073741824;
 
-        public string EndpointsPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "endpoints");
-            }
-        }
+        [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
+        public int NoDiscord { get; set; } = 0;
 
-        public string UserDataPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "users");
-            }
-        }
-
-        public string LogPath
-        {
-            get
-            {
-                return Path.Combine(DataPath, "logs");
-            }
-        }
+        public string EndpointsPath => Path.Combine(DataPath, "endpoints");
+        public string UserDataPath => Path.Combine(DataPath, "users");
+        public string LogPath => Path.Combine(DataPath, "logs");
+        public bool DebugNoDiscord => NoDiscord == 1;
     }
 }

Tools/BiblioTech/LoggingRoleDriver.cs

@@ -0,0 +1,24 @@
+using BiblioTech.Rewards;
+using DiscordRewards;
+using Logging;
+using Newtonsoft.Json;
+
+namespace BiblioTech
+{
+    public class LoggingRoleDriver : IDiscordRoleDriver
+    {
+        private readonly ILog log;
+
+        public LoggingRoleDriver(ILog log)
+        {
+            this.log = log;
+        }
+
+        public async Task GiveRewards(GiveRewardsCommand rewards)
+        {
+            await Task.CompletedTask;
+
+            log.Log(JsonConvert.SerializeObject(rewards, Formatting.None));
+        }
+    }
+}

Tools/BiblioTech/Program.cs

@@ -41,25 +41,15 @@ namespace BiblioTech
         public async Task MainAsync(string[] args)
         {
             Log.Log("Starting Codex Discord Bot...");
-            client = new DiscordSocketClient();
-            client.Log += ClientLog;
-
-            var notifyCommand = new NotifyCommand();
-            var associateCommand = new UserAssociateCommand(notifyCommand);
-            var sprCommand = new SprCommand();
-            var handler = new CommandHandler(client,
-                new GetBalanceCommand(associateCommand), 
-                new MintCommand(associateCommand),
-                sprCommand,
-                associateCommand,
-                notifyCommand,
-                new AdminCommand(sprCommand),
-                new MarketCommand()
-            );
-
-            await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
-            await client.StartAsync();
-            AdminChecker = new AdminChecker();
+            if (Config.DebugNoDiscord)
+            {
+                Log.Log("Debug option is set. Discord connection disabled!");
+                RoleDriver = new LoggingRoleDriver(Log);
+            }
+            else
+            {
+                await StartDiscordBot();
+            }
 
             var builder = WebApplication.CreateBuilder(args);
             builder.WebHost.ConfigureKestrel((context, options) =>
@@ -75,6 +65,29 @@ namespace BiblioTech
             await Task.Delay(-1);
         }
 
+        private async Task StartDiscordBot()
+        {
+            client = new DiscordSocketClient();
+            client.Log += ClientLog;
+
+            var notifyCommand = new NotifyCommand();
+            var associateCommand = new UserAssociateCommand(notifyCommand);
+            var sprCommand = new SprCommand();
+            var handler = new CommandHandler(client,
+                new GetBalanceCommand(associateCommand),
+                new MintCommand(associateCommand),
+                sprCommand,
+                associateCommand,
+                notifyCommand,
+                new AdminCommand(sprCommand),
+                new MarketCommand()
+            );
+
+            await client.LoginAsync(TokenType.Bot, Config.ApplicationToken);
+            await client.StartAsync();
+            AdminChecker = new AdminChecker();
+        }
+
         private static void PrintHelp()
         {
             Log.Log("BiblioTech - Codex Discord Bot");

Tools/CodexNetDeployer/CodexNodeStarter.cs

@@ -43,7 +43,7 @@ namespace CodexNetDeployer
                     {
                         s.EnableMarketplace(gethNode, contracts, m =>
                         {
-                            m.WithInitial(100.Eth(), config.InitialTestTokens.TestTokens());
+                            m.WithInitial(100.Eth(), config.InitialTestTokens.TstWei());
                             if (validatorsLeft > 0) m.AsValidator();
                             if (config.ShouldMakeStorageAvailable) m.AsStorageNode();
                         });
@@ -71,8 +71,8 @@ namespace CodexNetDeployer
                         var availability = new StorageAvailability(
                             totalSpace: config.StorageSell!.Value.MB(),
                             maxDuration: TimeSpan.FromSeconds(config.MaxDuration),
-                            minPriceForTotalSpace: config.MinPrice.TestTokens(),
-                            maxCollateral: config.MaxCollateral.TestTokens()
+                            minPriceForTotalSpace: config.MinPrice.TstWei(),
+                            maxCollateral: config.MaxCollateral.TstWei()
                         );
 
                         var response = codexNode.Marketplace.MakeStorageAvailable(availability);

Tools/KeyMaker/Controllers/KeyController.cs

@@ -0,0 +1,23 @@
+using GethPlugin;
+using Microsoft.AspNetCore.Mvc;
+
+namespace KeyMaker.Controllers
+{
+    [Route("api/[controller]")]
+    [ApiController]
+    public class KeyController : ControllerBase
+    {
+        [HttpGet]
+        public KeyResponse Get()
+        {
+            var account = EthAccount.GenerateNew();
+
+            return new KeyResponse
+            {
+                Public = account.EthAddress.Address,
+                Private = account.PrivateKey,
+                Secure = "Not Secure! For demo/development purposes only!"
+            };
+        }
+    }
+}

Tools/KeyMaker/KeyMaker.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.15" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/KeyMaker/KeyMaker.csproj.user

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugProfile>https</ActiveDebugProfile>
+  </PropertyGroup>
+</Project>

Tools/KeyMaker/KeyResponse.cs

@@ -0,0 +1,9 @@
+namespace KeyMaker
+{
+    public class KeyResponse
+    {
+        public string Public { get; set; } = string.Empty;
+        public string Private { get; set; } = string.Empty;
+        public string Secure { get; set; } = string.Empty;
+    }
+}

Tools/KeyMaker/Program.cs

@@ -0,0 +1,31 @@
+var builder = WebApplication.CreateBuilder(args);
+
+var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+builder.WebHost.ConfigureKestrel((context, options) =>
+{
+    options.ListenAnyIP(Convert.ToInt32(listenPort));
+});
+
+builder.Services.AddControllers();
+builder.Services.AddEndpointsApiExplorer();
+builder.Services.AddSwaggerGen();
+
+var app = builder.Build();
+
+if (app.Environment.IsDevelopment())
+{
+    app.UseSwagger();
+    app.UseSwaggerUI();
+}
+
+app.UseHttpsRedirection();
+
+app.UseAuthorization();
+
+app.MapControllers();
+
+Console.WriteLine("KeyMaker listening on port " + listenPort);
+
+app.Run();

Tools/KeyMaker/Properties/launchSettings.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json.schemastore.org/launchsettings.json",
+  "iisSettings": {
+    "windowsAuthentication": false,
+    "anonymousAuthentication": true,
+    "iisExpress": {
+      "applicationUrl": "http://localhost:17248",
+      "sslPort": 44396
+    }
+  },
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "http://localhost:5069",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "applicationUrl": "https://localhost:7056;http://localhost:5069",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    },
+    "IIS Express": {
+      "commandName": "IISExpress",
+      "launchBrowser": true,
+      "launchUrl": "swagger",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

Tools/KeyMaker/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

Tools/KeyMaker/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

Tools/KeyMaker/docker/Dockerfile

@@ -0,0 +1,26 @@
+# Variables
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG IMAGE=${BUILDER}
+ARG APP_HOME=/app
+
+
+# Build
+FROM ${IMAGE} AS builder
+ARG APP_HOME
+
+WORKDIR ${APP_HOME}
+COPY ./Tools/KeyMaker ./Tools/KeyMaker
+COPY ./Framework ./Framework
+COPY ./ProjectPlugins ./ProjectPlugins
+RUN dotnet restore Tools/KeyMaker
+RUN dotnet publish Tools/KeyMaker -c Release -o out
+
+
+# Create
+FROM ${IMAGE}
+ARG APP_HOME
+ENV APP_HOME=${APP_HOME}
+
+WORKDIR ${APP_HOME}
+COPY --from=builder ${APP_HOME}/out .
+CMD dotnet ${APP_HOME}/KeyMaker.dll

Tools/TestNetRewarder/BufferLogger.cs

@@ -0,0 +1,41 @@
+using Logging;
+
+namespace TestNetRewarder
+{
+    public class BufferLogger : ILog
+    {
+        private readonly List<string> lines = new List<string>();
+
+        public void AddStringReplace(string from, string to)
+        {
+            throw new NotImplementedException();
+        }
+
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Debug(string message = "", int skipFrames = 0)
+        {
+            lines.Add(message);
+        }
+
+        public void Error(string message)
+        {
+            lines.Add($"Error: {message}");
+        }
+
+        public void Log(string message)
+        {
+            lines.Add(message);
+        }
+
+        public string[] Get()
+        {
+            var result = lines.ToArray();
+            lines.Clear();
+            return result;
+        }
+    }
+}

Tools/TestNetRewarder/ChainChangeMux.cs

@@ -0,0 +1,45 @@
+using CodexContractsPlugin.ChainMonitor;
+using System.Numerics;
+
+namespace TestNetRewarder
+{
+    public class ChainChangeMux : IChainStateChangeHandler
+    {
+        private readonly IChainStateChangeHandler[] handlers;
+
+        public ChainChangeMux(params IChainStateChangeHandler[] handlers)
+        {
+            this.handlers = handlers;
+        }
+
+        public void OnNewRequest(IChainStateRequest request)
+        {
+            foreach (var handler in handlers) handler.OnNewRequest(request);
+        }
+
+        public void OnRequestCancelled(IChainStateRequest request)
+        {
+            foreach (var handler in handlers) handler.OnRequestCancelled(request);
+        }
+
+        public void OnRequestFinished(IChainStateRequest request)
+        {
+            foreach (var handler in handlers) handler.OnRequestFinished(request);
+        }
+
+        public void OnRequestFulfilled(IChainStateRequest request)
+        {
+            foreach (var handler in handlers) handler.OnRequestFulfilled(request);
+        }
+
+        public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex)
+        {
+            foreach (var handler in handlers) handler.OnSlotFilled(request, slotIndex);
+        }
+
+        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
+        {
+            foreach (var handler in handlers) handler.OnSlotFreed(request, slotIndex);
+        }
+    }
+}

Tools/TestNetRewarder/ChainState.cs

@@ -1,151 +0,0 @@
-using CodexContractsPlugin;
-using CodexContractsPlugin.Marketplace;
-using NethereumWorkflow.BlockUtils;
-using Newtonsoft.Json;
-using Utils;
-
-namespace TestNetRewarder
-{
-    public class ChainState
-    {
-        private readonly HistoricState historicState;
-        private readonly string[] colorIcons = new[]
-        {
-            "🔴",
-            "🟠",
-            "🟡",
-            "🟢",
-            "🔵",
-            "🟣",
-            "🟤",
-            "⚫",
-            "⚪",
-            "🟥",
-            "🟧",
-            "🟨",
-            "🟩",
-            "🟦",
-            "🟪",
-            "🟫",
-            "⬛",
-            "⬜",
-            "🔶",
-            "🔷"
-        };
-
-        public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange)
-        {
-            this.historicState = historicState;
-
-            NewRequests = contracts.GetStorageRequests(blockRange);
-            historicState.CleanUpOldRequests();
-            historicState.ProcessNewRequests(NewRequests);
-            historicState.UpdateStorageRequests(contracts);
-
-            StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray();
-            FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray();
-            RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange);
-            RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange);
-            SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange);
-            SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange);
-        }
-
-        public Request[] NewRequests { get; }
-        public StorageRequest[] AllRequests => historicState.StorageRequests;
-        public StorageRequest[] StartedRequests { get; private set; }
-        public StorageRequest[] FinishedRequests { get; private set; }
-        public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; }
-        public RequestCancelledEventDTO[] RequestCancelledEvents { get; }
-        public SlotFilledEventDTO[] SlotFilledEvents { get; }
-        public SlotFreedEventDTO[] SlotFreedEvents { get; }
-
-        public string[] GenerateOverview()
-        {
-            var entries = new List<StringBlockNumberPair>();
-
-            entries.AddRange(NewRequests.Select(ToPair));
-            entries.AddRange(RequestFulfilledEvents.Select(ToPair));
-            entries.AddRange(RequestCancelledEvents.Select(ToPair));
-            entries.AddRange(SlotFilledEvents.Select(ToPair));
-            entries.AddRange(SlotFreedEvents.Select(ToPair));
-            entries.AddRange(FinishedRequests.Select(ToPair));
-
-            entries.Sort(new StringUtcComparer());
-
-            return entries.Select(ToLine).ToArray();
-        }
-
-        private StringBlockNumberPair ToPair(Request r)
-        {
-            return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
-        }
-
-        public StringBlockNumberPair ToPair(StorageRequest r)
-        {
-            return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId);
-        }
-
-        private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r)
-        {
-            return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
-        }
-
-        private StringBlockNumberPair ToPair(RequestCancelledEventDTO r)
-        {
-            return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
-        }
-
-        private StringBlockNumberPair ToPair(SlotFilledEventDTO r)
-        {
-            return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
-        }
-
-        private StringBlockNumberPair ToPair(SlotFreedEventDTO r)
-        {
-            return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId);
-        }
-
-        private string ToLine(StringBlockNumberPair pair)
-        {
-            var nl = Environment.NewLine;
-            var colorIcon = GetColorIcon(pair.RequestId);
-            return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" +
-                $"```json{nl}" +
-                $"{pair.Str}{nl}" +
-                $"```";
-        }
-
-        private string GetColorIcon(byte[] requestId)
-        {
-            var index = requestId[0] % colorIcons.Length;
-            return colorIcons[index];
-        }
-
-        public class StringBlockNumberPair
-        {
-            public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId)
-            {
-                Name = name;
-                Str = str;
-                Block = block;
-                RequestId = requestId;
-            }
-
-            public string Name { get; }
-            public string Str { get; }
-            public BlockTimeEntry Block { get; }
-            public byte[] RequestId { get; }
-        }
-
-        public class StringUtcComparer : IComparer<StringBlockNumberPair>
-        {
-            public int Compare(StringBlockNumberPair? x, StringBlockNumberPair? y)
-            {
-                if (x == null && y == null) return 0;
-                if (x == null) return 1;
-                if (y == null) return -1;
-                return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber);
-            }
-        }
-    }
-}

Tools/TestNetRewarder/Checks.cs

@@ -1,141 +0,0 @@
-using CodexContractsPlugin.Marketplace;
-using GethPlugin;
-using NethereumWorkflow;
-using Utils;
-
-namespace TestNetRewarder
-{
-    public interface ICheck
-    {
-        EthAddress[] Check(ChainState state); 
-    }
-
-    public class FilledAnySlotCheck : ICheck
-    {
-        public EthAddress[] Check(ChainState state)
-        {
-            return state.SlotFilledEvents.Select(e => e.Host).ToArray();
-        }
-    }
-
-    public class FinishedSlotCheck : ICheck
-    {
-        private readonly ByteSize minSize;
-        private readonly TimeSpan minDuration;
-
-        public FinishedSlotCheck(ByteSize minSize, TimeSpan minDuration)
-        {
-            this.minSize = minSize;
-            this.minDuration = minDuration;
-        }
-
-        public EthAddress[] Check(ChainState state)
-        {
-            return state.FinishedRequests
-                .Where(r =>
-                    MeetsSizeRequirement(r) &&
-                    MeetsDurationRequirement(r))
-                .SelectMany(r => r.Hosts)
-                .ToArray();
-        }
-
-        private bool MeetsSizeRequirement(StorageRequest r)
-        {
-            var slotSize = r.Request.Ask.SlotSize.ToDecimal();
-            decimal min = minSize.SizeInBytes;
-            return slotSize >= min;
-        }
-
-        private bool MeetsDurationRequirement(StorageRequest r)
-        {
-            var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration);
-            return duration >= minDuration;
-        }
-    }
-
-    public class PostedContractCheck : ICheck
-    {
-        private readonly ulong minNumberOfHosts;
-        private readonly ByteSize minSlotSize;
-        private readonly TimeSpan minDuration;
-
-        public PostedContractCheck(ulong minNumberOfHosts, ByteSize minSlotSize, TimeSpan minDuration)
-        {
-            this.minNumberOfHosts = minNumberOfHosts;
-            this.minSlotSize = minSlotSize;
-            this.minDuration = minDuration;
-        }
-
-        public EthAddress[] Check(ChainState state)
-        {
-            return state.NewRequests
-                .Where(r =>
-                    MeetsNumSlotsRequirement(r) &&
-                    MeetsSizeRequirement(r) &&
-                    MeetsDurationRequirement(r))
-                .Select(r => r.ClientAddress)
-                .ToArray();
-        }
-
-        private bool MeetsNumSlotsRequirement(Request r)
-        {
-            return r.Ask.Slots >= minNumberOfHosts;
-        }
-
-        private bool MeetsSizeRequirement(Request r)
-        {
-            var slotSize = r.Ask.SlotSize.ToDecimal();
-            decimal min = minSlotSize.SizeInBytes;
-            return slotSize >= min;
-        }
-
-        private bool MeetsDurationRequirement(Request r)
-        {
-            var duration = TimeSpan.FromSeconds((double)r.Ask.Duration);
-            return duration >= minDuration;
-        }
-    }
-
-    public class StartedContractCheck : ICheck
-    {
-        private readonly ulong minNumberOfHosts;
-        private readonly ByteSize minSlotSize;
-        private readonly TimeSpan minDuration;
-
-        public StartedContractCheck(ulong minNumberOfHosts, ByteSize minSlotSize, TimeSpan minDuration)
-        {
-            this.minNumberOfHosts = minNumberOfHosts;
-            this.minSlotSize = minSlotSize;
-            this.minDuration = minDuration;
-        }
-
-        public EthAddress[] Check(ChainState state)
-        {
-            return state.StartedRequests
-                .Where(r =>
-                    MeetsNumSlotsRequirement(r) &&
-                    MeetsSizeRequirement(r) &&
-                    MeetsDurationRequirement(r))
-                .Select(r => r.Request.ClientAddress)
-                .ToArray();
-        }
-
-        private bool MeetsNumSlotsRequirement(StorageRequest r)
-        {
-            return r.Request.Ask.Slots >= minNumberOfHosts;
-        }
-
-        private bool MeetsSizeRequirement(StorageRequest r)
-        {
-            var slotSize = r.Request.Ask.SlotSize.ToDecimal();
-            decimal min = minSlotSize.SizeInBytes;
-            return slotSize >= min;
-        }
-
-        private bool MeetsDurationRequirement(StorageRequest r)
-        {
-            var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration);
-            return duration >= minDuration;
-        }
-    }
-}

Tools/TestNetRewarder/Configuration.cs

@@ -40,5 +40,14 @@ namespace TestNetRewarder
                 return TimeSpan.FromMinutes(IntervalMinutes);
             }
         }
+
+        public DateTime HistoryStartUtc
+        {
+            get
+            {
+                if (CheckHistoryTimestamp == 0) throw new Exception("'check-history' unix timestamp is required. Set it to the start/launch moment of the testnet.");
+                return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime;
+            }
+        }
     }
 }

Tools/TestNetRewarder/HistoricState.cs

@@ -29,6 +29,20 @@ namespace TestNetRewarder
                 r.State == RequestState.Failed
             );
         }
+
+        public string EntireString()
+        {
+            return JsonConvert.SerializeObject(StorageRequests);
+        }
+
+        public HistoricState()
+        {
+        }
+
+        public HistoricState(StorageRequest[] requests)
+        {
+            storageRequests.AddRange(requests);
+        }
     }
 
     public class StorageRequest

Tools/TestNetRewarder/MarketBuffer.cs

@@ -0,0 +1,74 @@
+using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
+using DiscordRewards;
+using System.Numerics;
+
+namespace TestNetRewarder
+{
+    public class MarketBuffer
+    {
+        private readonly List<IChainStateRequest> requests = new List<IChainStateRequest>();
+        private readonly TimeSpan bufferSpan;
+
+        public MarketBuffer(TimeSpan bufferSpan)
+        {
+            this.bufferSpan = bufferSpan;
+        }
+
+        public void Add(IChainStateRequest request)
+        {
+            requests.Add(request);
+        }
+
+        public void Update()
+        {
+            var now = DateTime.UtcNow;
+            requests.RemoveAll(r => (now - r.FinishedUtc) > bufferSpan);
+        }
+
+        public MarketAverage? GetAverage()
+        {
+            if (requests.Count == 0) return null;
+
+            return new MarketAverage
+            {
+                NumberOfFinished = requests.Count,
+                TimeRangeSeconds = (int)bufferSpan.TotalSeconds,
+                Price = Average(s => s.Request.Ask.Reward),
+                Duration = Average(s => s.Request.Ask.Duration),
+                Size = Average(s => GetTotalSize(s.Request.Ask)),
+                Collateral = Average(s => s.Request.Ask.Collateral),
+                ProofProbability = Average(s => s.Request.Ask.ProofProbability)
+            };
+        }
+
+        private float Average(Func<IChainStateRequest, BigInteger> getValue)
+        {
+            return Average(s =>
+            {
+                var value = getValue(s);
+                return (int)value;
+            });
+        }
+
+        private float Average(Func<IChainStateRequest, int> getValue)
+        {
+            var sum = 0.0f;
+            float count = requests.Count;
+            foreach (var r in requests)
+            {
+                sum += getValue(r);
+            }
+
+            if (count < 1.0f) return 0.0f;
+            return sum / count;
+        }
+
+        private int GetTotalSize(Ask ask)
+        {
+            var nSlots = Convert.ToInt32(ask.Slots);
+            var slotSize = (int)ask.SlotSize;
+            return nSlots * slotSize;
+        }
+    }
+}

Tools/TestNetRewarder/MarketTracker.cs

@@ -1,124 +1,73 @@
-using CodexContractsPlugin.Marketplace;
+using CodexContractsPlugin.ChainMonitor;
 using DiscordRewards;
+using Logging;
 using System.Numerics;
 
 namespace TestNetRewarder
 {
-    public class MarketTracker
+    public class MarketTracker : IChainStateChangeHandler
     {
-        private readonly List<ChainState> buffer = new List<ChainState>();
+        private readonly List<MarketBuffer> buffers = new List<MarketBuffer>();
+        private readonly ILog log;
 
-        public MarketAverage[] ProcessChainState(ChainState chainState)
+        public MarketTracker(Configuration config, ILog log)
         {
-            var intervalCounts = GetInsightCounts();
-            if (!intervalCounts.Any()) return Array.Empty<MarketAverage>();
+            var intervals = GetInsightCounts(config);
 
-            UpdateBuffer(chainState, intervalCounts.Max());
-            var result = intervalCounts
-                .Select(GenerateMarketAverage)
-                .Where(a => a != null)
-                .Cast<MarketAverage>()
-                .ToArray();
-
-            if (!result.Any()) result = Array.Empty<MarketAverage>();
-            return result;
-        }
-
-        private void UpdateBuffer(ChainState chainState, int maxNumberOfIntervals)
-        {
-            buffer.Add(chainState);
-            while (buffer.Count > maxNumberOfIntervals)
+            foreach (var i in intervals)
             {
-                buffer.RemoveAt(0);
+                buffers.Add(new MarketBuffer(
+                    config.Interval * i
+                ));
             }
+
+            this.log = log;
         }
 
-        private MarketAverage? GenerateMarketAverage(int numberOfIntervals)
+        public MarketAverage[] GetAverages()
         {
-            var states = SelectStates(numberOfIntervals);
-            return CreateAverage(states);
+            foreach (var b in buffers) b.Update();
+
+            return buffers.Select(b => b.GetAverage()).Where(a => a != null).Cast<MarketAverage>().ToArray();
         }
 
-        private ChainState[] SelectStates(int numberOfIntervals)
+        public void OnNewRequest(IChainStateRequest request)
         {
-            if (numberOfIntervals < 1) return Array.Empty<ChainState>();
-            if (numberOfIntervals > buffer.Count) return Array.Empty<ChainState>();
-            return buffer.TakeLast(numberOfIntervals).ToArray();
         }
 
-        private MarketAverage? CreateAverage(ChainState[] states)
+        public void OnRequestFinished(IChainStateRequest request)
+        {
+            foreach (var b in buffers) b.Add(request);
+        }
+
+        public void OnRequestFulfilled(IChainStateRequest request)
+        {
+        }
+
+        public void OnRequestCancelled(IChainStateRequest request)
+        {
+        }
+
+        public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
+        {
+        }
+
+        private int[] GetInsightCounts(Configuration config)
         {
             try
             {
-                return new MarketAverage
-                {
-                    NumberOfFinished = CountNumberOfFinishedRequests(states),
-                    TimeRangeSeconds = GetTotalTimeRange(states),
-                    Price = Average(states, s => s.Request.Ask.Reward),
-                    Duration = Average(states, s => s.Request.Ask.Duration),
-                    Size = Average(states, s => GetTotalSize(s.Request.Ask)),
-                    Collateral = Average(states, s => s.Request.Ask.Collateral),
-                    ProofProbability = Average(states, s => s.Request.Ask.ProofProbability)
-                };
-            }
-            catch (Exception ex)
-            {
-                Program.Log.Error($"Exception in CreateAverage: {ex}");
-                return null;
-            }
-        }
-
-        private int GetTotalSize(Ask ask)
-        {
-            var nSlots = Convert.ToInt32(ask.Slots);
-            var slotSize = Convert.ToInt32(ask.SlotSize);
-            return nSlots * slotSize;
-        }
-
-        private float Average(ChainState[] states, Func<StorageRequest, BigInteger> getValue)
-        {
-            return Average(states, s => Convert.ToInt32(getValue(s)));
-        }
-
-        private float Average(ChainState[] states, Func<StorageRequest, int> getValue)
-        {
-            var sum = 0.0f;
-            var count = 0.0f;
-            foreach (var state in states)
-            {
-                foreach (var finishedRequest in state.FinishedRequests)
-                {
-                    sum += getValue(finishedRequest);
-                    count++;
-                }
-            }
-
-            if (count < 1.0f) return 0.0f;
-            return sum / count;
-        }
-
-        private int GetTotalTimeRange(ChainState[] states)
-        {
-            return Convert.ToInt32((Program.Config.Interval * states.Length).TotalSeconds);
-        }
-
-        private int CountNumberOfFinishedRequests(ChainState[] states)
-        {
-            return states.Sum(s => s.FinishedRequests.Length);
-        }
-
-        private int[] GetInsightCounts()
-        {
-            try
-            {
-                var tokens = Program.Config.MarketInsights.Split(';').ToArray();
+                var tokens = config.MarketInsights.Split(';').ToArray();
                 return tokens.Select(t => Convert.ToInt32(t)).ToArray();
             }
             catch (Exception ex)
             {
-                Program.Log.Error($"Exception when parsing MarketInsights config parameters: {ex}");
+                log.Error($"Exception when parsing MarketInsights config parameters: {ex}");
             }
-            return Array.Empty<int>();            
+            return Array.Empty<int>();
         }
     }
 }

Tools/TestNetRewarder/Processor.cs

@@ -1,146 +1,67 @@
-using DiscordRewards;
-using GethPlugin;
+using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
 using Logging;
-using Newtonsoft.Json;
 using Utils;
 
 namespace TestNetRewarder
 {
-    public class Processor
+    public class Processor : ITimeSegmentHandler
     {
-        private static readonly HistoricState historicState = new HistoricState();
-        private static readonly RewardRepo rewardRepo = new RewardRepo();
-        private static readonly MarketTracker marketTracker = new MarketTracker();
+        private readonly RequestBuilder builder;
+        private readonly RewardChecker rewardChecker;
+        private readonly MarketTracker marketTracker;
+        private readonly BufferLogger bufferLogger;
+        private readonly ChainState chainState;
+        private readonly BotClient client;
         private readonly ILog log;
-        private BlockInterval? lastBlockRange;
 
-        public Processor(ILog log)
+        public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log)
         {
+            this.client = client;
             this.log = log;
+
+            builder = new RequestBuilder();
+            rewardChecker = new RewardChecker(builder);
+            marketTracker = new MarketTracker(config, log);
+            bufferLogger = new BufferLogger();
+
+            var handler = new ChainChangeMux(
+                rewardChecker.Handler,
+                marketTracker
+            );
+
+            chainState = new ChainState(new LogSplitter(log, bufferLogger), contracts, handler, config.HistoryStartUtc);
         }
 
-        public async Task ProcessTimeSegment(TimeRange timeRange)
+        public async Task OnNewSegment(TimeRange timeRange)
         {
-            var connector = GethConnector.GethConnector.Initialize(log);
-            if (connector == null) throw new Exception("Invalid Geth information");
-
             try
             {
-                var blockRange = connector.GethNode.ConvertTimeRangeToBlockRange(timeRange);
-                if (!IsNewBlockRange(blockRange))
-                {
-                    log.Log($"Block range {blockRange} was previously processed. Skipping...");
-                    return;
-                }
+                chainState.Update(timeRange.To);
 
-                var chainState = new ChainState(historicState, connector.CodexContracts, blockRange);
-                await ProcessChainState(chainState);
+                var averages = marketTracker.GetAverages();
+                var lines = RemoveFirstLine(bufferLogger.Get());
+
+                var request = builder.Build(averages, lines);
+                if (request.HasAny())
+                {
+                    await client.SendRewards(request);
+                }
             }
             catch (Exception ex)
             {
-                log.Error("Exception processing time segment: " + ex);
+                var msg = "Exception processing time segment: " + ex;
+                log.Error(msg); 
+                bufferLogger.Error(msg);
                 throw;
             }
         }
 
-        private bool IsNewBlockRange(BlockInterval blockRange)
+        private string[] RemoveFirstLine(string[] lines)
         {
-            if (lastBlockRange == null ||
-                lastBlockRange.From != blockRange.From || 
-                lastBlockRange.To != blockRange.To)
-            {
-                lastBlockRange = blockRange;
-                return true;
-            }
-
-            return false;
-        }
-
-        private async Task ProcessChainState(ChainState chainState)
-        {
-            var outgoingRewards = new List<RewardUsersCommand>();
-            foreach (var reward in rewardRepo.Rewards)
-            {
-                ProcessReward(outgoingRewards, reward, chainState);
-            }
-
-            var marketAverages = GetMarketAverages(chainState);
-            var eventsOverview = GenerateEventsOverview(chainState);
-
-            log.Log($"Found {outgoingRewards.Count} rewards. " +
-                $"Found {marketAverages.Length} market averages. " +
-                $"Found {eventsOverview.Length} events.");
-
-            if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any())
-            {
-                if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview))
-                {
-                    log.Error("Failed to send reward command.");
-                }
-            }
-        }
-
-        private string[] GenerateEventsOverview(ChainState chainState)
-        {
-            return chainState.GenerateOverview();
-        }
-
-        private MarketAverage[] GetMarketAverages(ChainState chainState)
-        {
-            return marketTracker.ProcessChainState(chainState);
-        }
-
-        private async Task<bool> SendRewardsCommand(List<RewardUsersCommand> outgoingRewards, MarketAverage[] marketAverages, string[] eventsOverview)
-        {
-            var cmd = new GiveRewardsCommand
-            {
-                Rewards = outgoingRewards.ToArray(),
-                Averages = marketAverages.ToArray(),
-                EventsOverview = eventsOverview
-            };
-
-            log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd));
-            return await Program.BotClient.SendRewards(cmd);
-        }
-
-        private void ProcessReward(List<RewardUsersCommand> outgoingRewards, RewardConfig reward, ChainState chainState)
-        {
-            var winningAddresses = PerformCheck(reward, chainState);
-            foreach (var win in winningAddresses)
-            {
-                log.Log($"Address '{win.Address}' wins '{reward.Message}'");
-            }
-            if (winningAddresses.Any())
-            {
-                outgoingRewards.Add(new RewardUsersCommand
-                {
-                    RewardId = reward.RoleId,
-                    UserAddresses = winningAddresses.Select(a => a.Address).ToArray()
-                });
-            }
-        }
-
-        private EthAddress[] PerformCheck(RewardConfig reward, ChainState chainState)
-        {
-            var check = GetCheck(reward.CheckConfig);
-            return check.Check(chainState).Distinct().ToArray();
-        }
-
-        private ICheck GetCheck(CheckConfig config)
-        {
-            switch (config.Type)
-            {
-                case CheckType.FilledSlot:
-                    return new FilledAnySlotCheck();
-                case CheckType.FinishedSlot:
-                    return new FinishedSlotCheck(config.MinSlotSize, config.MinDuration);
-                case CheckType.PostedContract:
-                    return new PostedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration);
-                case CheckType.StartedContract:
-                    return new StartedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration);
-            }
-
-            throw new Exception("Unknown check type: " + config.Type);
+            //if (!lines.Any()) return Array.Empty<string>();
+            //return lines.Skip(1).ToArray();
+            return lines;
         }
     }
 }

Tools/TestNetRewarder/Program.cs

@@ -6,10 +6,10 @@ namespace TestNetRewarder
 {
     public class Program
     {
-        public static Configuration Config { get; private set; } = null!;
-        public static ILog Log { get; private set; } = null!;
-        public static CancellationToken CancellationToken { get; private set; }
-        public static BotClient BotClient { get; private set; } = null!;
+        public static CancellationToken CancellationToken;
+        private static Configuration Config = null!;
+        private static ILog Log = null!;
+        private static BotClient BotClient = null!;
         private static Processor processor = null!;
         private static DateTime lastCheck = DateTime.MinValue;
 
@@ -27,8 +27,11 @@ namespace TestNetRewarder
                 new ConsoleLog()
             );
 
+            var connector = GethConnector.GethConnector.Initialize(Log);
+            if (connector == null) throw new Exception("Invalid Geth information");
+
             BotClient = new BotClient(Config, Log);
-            processor = new Processor(Log);
+            processor = new Processor(Config, BotClient, connector.CodexContracts, Log);
 
             EnsurePath(Config.DataPath);
             EnsurePath(Config.LogPath);
@@ -41,12 +44,12 @@ namespace TestNetRewarder
             EnsureGethOnline();
 
             Log.Log("Starting TestNet Rewarder...");
-            var segmenter = new TimeSegmenter(Log, Config);
+            var segmenter = new TimeSegmenter(Log, Config, processor);
          
             while (!CancellationToken.IsCancellationRequested)
             {
                 await EnsureBotOnline();
-                await segmenter.WaitForNextSegment(processor.ProcessTimeSegment);
+                await segmenter.ProcessNextSegment();
                 await Task.Delay(100, CancellationToken);
             }
         }