Creates new overview after 3 new changes.

Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/KubernetesWorkflow/Configuration.cs

@@ -21,8 +21,5 @@ namespace KubernetesWorkflow
 
         [JsonIgnore]
         public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
-
-        [JsonIgnore]
-        public Func<string?, string?> Replacer { get; set; } = s => s;
     }
 }

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,13 +11,11 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
-        private readonly Func<string?, string?> replacer;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace,
-            Func<string?, string?> replacer)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
@@ -25,7 +23,6 @@ namespace KubernetesWorkflow
             this.podName = podName;
             this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
-            this.replacer = replacer;
             cts = new CancellationTokenSource();
         }
 
@@ -95,7 +92,7 @@ namespace KubernetesWorkflow
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
             var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
-            handler.Log(stream, replacer);
+            handler.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -12,11 +12,10 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly K8sCluster cluster;
         private readonly WorkflowNumberSource workflowNumberSource;
-        private readonly Func<string?, string?> replacer;
         private readonly K8sClient client;
         public const string PodLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace, Func<string?, string?> replacer)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
         {
             this.log = log;
             this.cluster = cluster;
@@ -24,7 +23,6 @@ namespace KubernetesWorkflow
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sNamespace = k8sNamespace;
-            this.replacer = replacer;
         }
 
         public void Dispose()
@@ -66,7 +64,7 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous, Func<string?, string?> replacer)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
         {
             log.Debug();
 
@@ -74,7 +72,7 @@ namespace KubernetesWorkflow
             var recipeName = container.Recipe.Name;
 
             using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
-            logHandler.Log(stream, replacer);
+            logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -908,7 +906,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true, replacer);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }
@@ -954,7 +952,7 @@ namespace KubernetesWorkflow
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace, replacer);
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/LogHandler.cs

@@ -4,19 +4,19 @@ namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
-        void Log(Stream log, Func<string?, string?> replacer);
+        void Log(Stream log);
     }
 
     public abstract class LogHandler : ILogHandler
     {
-        public void Log(Stream log, Func<string?, string?> replacer)
+        public void Log(Stream log)
         {
             using var reader = new StreamReader(log);
             var line = reader.ReadLine();
             while (line != null)
             {
-                line = replacer(reader.ReadLine());
-                if (line != null) ProcessLine(line);
+                ProcessLine(line);
+                line = reader.ReadLine();
             }
         }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -28,17 +28,16 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly string k8sNamespace;
-        private readonly Func<string?, string?> replacer;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace, Func<string?, string?> replacer)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.k8sNamespace = k8sNamespace;
-            this.replacer = replacer;
+
             locationProvider = new LocationProvider(log, K8s);
         }
 
@@ -120,7 +119,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
         }
 
@@ -132,7 +131,7 @@ namespace KubernetesWorkflow
 
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
 
             return new DownloadedLog(logHandler, container.Name);
@@ -258,7 +257,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
                 action(controller);
                 controller.Dispose();
             }
@@ -273,7 +272,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
                 var result = action(controller);
                 controller.Dispose();
                 return result;

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride), configuration.Replacer);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -5,6 +5,16 @@
         private static readonly Random random = new Random();
         private static readonly object @lock = new object();
 
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
+
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
             lock (@lock)

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -66,13 +69,18 @@ namespace CodexContractsPlugin.ChainMonitor
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;
@@ -109,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -117,7 +125,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -125,7 +133,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -133,7 +141,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -142,13 +150,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -162,10 +178,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
 using NethereumWorkflow.BlockUtils;
 using Utils;
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-fb25372";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "6B-94-24-A4-D5-01-6F-12-E9-34-74-36-80-57-7A-3A-79-8C-E8-02-68-B7-05-DA-50-A0-5C-B1-02-B9-AE-C6";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -63,17 +63,17 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -82,31 +82,44 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
         public StorageAvailability[] GetAvailabilities()
         {
-            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
             return mapper.Map(collection);
         }
 
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -261,4 +274,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "thatbenbierens/nim-codex:netpeerdebug6";
-            //"codexstorage/nim-codex:0.1.4";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -110,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -18,6 +18,7 @@ namespace CodexPlugin
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
@@ -138,6 +139,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +151,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
@@ -264,10 +271,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,16 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
+            };
+        }
+
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
             };
         }
 
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -136,47 +153,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -184,8 +199,20 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -245,4 +272,42 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -40,6 +40,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -83,33 +83,46 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
+
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
 
     DebugInfo:
       type: object
@@ -125,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -327,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -340,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -351,15 +384,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -425,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:
@@ -476,10 +526,36 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
+    post:
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetwork
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -502,6 +578,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -747,7 +849,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -765,7 +867,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -813,4 +915,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -7,4 +7,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

Tests/CodexTests/BasicTests/AsyncProfiling.cs

@@ -0,0 +1,85 @@
+using NUnit.Framework;
+using MetricsPlugin;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class AsyncProfiling : CodexDistTest
+    {
+        [Test]
+        public void AsyncProfileMetricsPlz()
+        {
+            var node = StartCodex(s => s.EnableMetrics());
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single();
+
+            var file = GenerateTestFile(100.MB());
+            node.UploadFile(file);
+
+            Thread.Sleep(10000);
+
+            var profilerMetrics = new AsyncProfileMetrics(metrics.GetAllMetrics());
+
+            var log = GetTestLog();
+            log.Log($"{nameof(profilerMetrics.CallCount)} = {profilerMetrics.CallCount.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTime)} = {profilerMetrics.ExecTime.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTimeWithChildren)} = {profilerMetrics.ExecTimeWithChildren.Highest()}");
+            log.Log($"{nameof(profilerMetrics.SingleExecTimeMax)} = {profilerMetrics.SingleExecTimeMax.Highest()}");
+            log.Log($"{nameof(profilerMetrics.WallTime)} = {profilerMetrics.WallTime.Highest()}");
+        }
+    }
+
+    public class AsyncProfileMetrics
+    {
+        public AsyncProfileMetrics(Metrics metrics)
+        {
+            CallCount = CreateMetric(metrics, "chronos_call_count_total");
+            ExecTime = CreateMetric(metrics, "chronos_exec_time_total");
+            ExecTimeWithChildren = CreateMetric(metrics, "chronos_exec_time_with_children_total");
+            SingleExecTimeMax = CreateMetric(metrics, "chronos_single_exec_time_max");
+            WallTime = CreateMetric(metrics, "chronos_wall_time_total");
+        }
+
+        public AsyncProfileMetric CallCount { get; }
+        public AsyncProfileMetric ExecTime { get; }
+        public AsyncProfileMetric ExecTimeWithChildren { get; }
+        public AsyncProfileMetric SingleExecTimeMax { get; }
+        public AsyncProfileMetric WallTime { get; }
+
+        private static AsyncProfileMetric CreateMetric(Metrics metrics, string name)
+        {
+            var sets = metrics.Sets.Where(s => s.Name == name).ToArray();
+            return new AsyncProfileMetric(sets);
+        }
+    }
+
+    public class AsyncProfileMetric
+    {
+        private readonly MetricsSet[] metricsSets;
+
+        public AsyncProfileMetric(MetricsSet[] metricsSets)
+        {
+            this.metricsSets = metricsSets;
+        }
+
+        public MetricsSet Highest()
+        {
+            MetricsSet? result = null;
+            var highest = double.MinValue;
+            foreach (var metric in metricsSets)
+            {
+                foreach (var value in metric.Values)
+                {
+                    if (value.Value > highest)
+                    {
+                        highest = value.Value;
+                        result = metric;
+                    }
+                }
+            }
+
+            if (result == null) throw new Exception("None were highest");
+            return result;
+        }
+    }
+}

Tests/CodexTests/BasicTests/EventLogginHandler.cs

@@ -1,57 +0,0 @@
-using CodexContractsPlugin.ChainMonitor;
-using GethPlugin;
-using Logging;
-using System.Numerics;
-
-namespace CodexTests.BasicTests
-{
-    public class EventLogginHandler : IChainStateChangeHandler
-    {
-        private readonly ILog log;
-
-        public EventLogginHandler(ILog log)
-        {
-            this.log = log;
-        }
-
-        public void OnNewRequest(RequestEvent requestEvent)
-        {
-            Log(nameof(OnNewRequest), requestEvent);
-        }
-
-        public void OnRequestCancelled(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestCancelled), requestEvent);
-        }
-
-        public void OnRequestFailed(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFailed), requestEvent);
-        }
-
-        public void OnRequestFinished(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFinished), requestEvent);
-        }
-
-        public void OnRequestFulfilled(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFulfilled), requestEvent);
-        }
-
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
-        {
-            Log(nameof(OnSlotFilled), requestEvent, host.ToString(), slotIndex.ToString());
-        }
-
-        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            Log(nameof(OnNewRequest), requestEvent, slotIndex.ToString());
-        }
-
-        private void Log(string name, object o, params string[] str)
-        {
-            log.Log(name + ": " + o.ToString() + " - " + string.Join(",", str));
-        }
-    }
-}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -2,7 +2,6 @@
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
-using Nethereum.JsonRpc.Client;
 using NUnit.Framework;
 using Utils;
 
@@ -11,18 +10,6 @@ namespace CodexTests.BasicTests
     [TestFixture]
     public class ExampleTests : CodexDistTest
     {
-        [Test]
-        public void A()
-        {
-            var oneMb = GenerateTestFile(1.MB(), "oneMB");
-            var fiveMb = GenerateTestFile(5.MB(), "fiveMb");
-            var tenMb = GenerateTestFile(10.MB(), "tenMb");
-            var hundredMb = GenerateTestFile(100.MB(), "hundredMb");
-            var oneGb = GenerateTestFile(1.GB(), "oneGb");
-
-            var a = 0;
-        }
-
         [Test]
         public void CodexLogExample()
         {
@@ -33,41 +20,11 @@ namespace CodexTests.BasicTests
             var localDatasets = primary.LocalFiles();
             CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
 
-            var nameMap = new Dictionary<string, string>();
-            AddNameMapping(nameMap, primary);
-
-            Get().Replacer = line =>
-            {
-                if (line == null) return null;
-                foreach (var pair in nameMap)
-                {
-                    line = line.Replace(pair.Key, pair.Value);
-                }
-                return line;
-            };
-
-
             var log = Ci.DownloadLog(primary);
 
             log.AssertLogContains("Uploaded file");
         }
 
-
-        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
-        {
-            var name = node.GetName();
-            var info = node.GetDebugInfo();
-            var nodeId = info.Table.LocalNode.NodeId;
-            var peerId = info.Table.LocalNode.PeerId;
-
-            nameMap.Add(nodeId, name);
-            nameMap.Add(peerId, name);
-            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
-            nameMap.Add(CodexUtils.ToShortId(peerId), name);
-            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
-            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
-        }
-
         [Test]
         public void TwoMetricsExample()
         {
@@ -79,7 +36,7 @@ namespace CodexTests.BasicTests
             var primary2 = group2[0];
             var secondary2 = group2[1];
 
-            var metrics = Ci.GetMetricsFor(primary, primary2);
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(10), primary, primary2);
 
             primary.ConnectToPeer(secondary);
             primary2.ConnectToPeer(secondary2);

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -1,5 +1,4 @@
 using CodexContractsPlugin;
-using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
 using FileUtils;
@@ -35,7 +34,10 @@ namespace CodexTests.BasicTests
             var numberOfHosts = 5;
             var hosts = StartCodex(numberOfHosts, s => s
                 .WithName("Host")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Info))
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                {
+                    ContractClock = CodexLogLevel.Trace,
+                })
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), hostInitialBalance)
@@ -111,104 +113,6 @@ namespace CodexTests.BasicTests
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
         }
 
-        [Test]
-        [Combinatorial]
-        public void FindBug(
-            [Values(64)] int numBlocks,
-            [Values(0)] int plusSizeKb,
-            [Values(0)] int plusSizeBytes
-        )
-        {
-            var numberOfHosts = 15;
-
-            var hostInitialBalance = 234.Tst();
-            var clientInitialBalance = 100000.Tst();
-            var fileSize = new ByteSize(
-                numBlocks * (64 * 1024) +
-                plusSizeKb * 1024 +
-                plusSizeBytes
-            );
-
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
-
-            var hosts = StartCodex(numberOfHosts, s => s
-                .WithName("Host")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Trace))
-                .WithStorageQuota(11.GB())
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), hostInitialBalance)
-                    .AsStorageNode()
-                    .AsValidator()));
-
-            foreach (var host in hosts)
-            {
-                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
-
-                var availability = new StorageAvailability(
-                    totalSpace: 10.GB(),
-                    maxDuration: TimeSpan.FromMinutes(30),
-                    minPriceForTotalSpace: 1.TstWei(),
-                    maxCollateral: 20.TstWei()
-                );
-                host.Marketplace.MakeStorageAvailable(availability);
-            }
-
-            var client = StartCodex(s => s
-                .WithName("Client")
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), clientInitialBalance)));
-
-            var nameMap = new Dictionary<string, string>();
-            AddNameMapping(nameMap, client);
-            foreach (var host in hosts) AddNameMapping(nameMap, host);
-
-            Get().Replacer = line =>
-            {
-                if (line == null) return null;
-                foreach (var pair in nameMap)
-                {
-                    line = line.Replace(pair.Key, pair.Value);
-                }
-                return line;
-            };
-
-            while (true)
-            {
-                var testFile = CreateFile(fileSize);
-                var uploadCid = client.UploadFile(testFile);
-
-                var purchase = new StoragePurchaseRequest(uploadCid)
-                {
-                    PricePerSlotPerSecond = 2.TstWei(),
-                    RequiredCollateral = 10.TstWei(),
-                    MinRequiredNumberOfNodes = 5,
-                    NodeFailureTolerance = 2,
-                    ProofProbability = 5,
-                    Duration = TimeSpan.FromMinutes(20),
-                    Expiry = TimeSpan.FromMinutes(10)
-                };
-
-                var purchaseContract = client.Marketplace.RequestStorage(purchase);
-                purchaseContract.WaitForStorageContractStarted();
-            }
-        }
-
-        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
-        {
-            var name = node.GetName();
-            var info = node.GetDebugInfo();
-            var nodeId = info.Table.LocalNode.NodeId;
-            var peerId = info.Table.LocalNode.PeerId;
-
-            nameMap.Add(nodeId, name);
-            nameMap.Add(peerId, name);
-            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
-            nameMap.Add(CodexUtils.ToShortId(peerId), name);
-            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
-            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
-        }
-
         private TrackedFile CreateFile(ByteSize fileSize)
         {
             var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,5 +1,7 @@
 using CodexPlugin;
+using FileUtils;
 using NUnit.Framework;
+using System.Diagnostics;
 using Utils;
 
 namespace CodexTests.BasicTests
@@ -10,11 +12,44 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
+        }
+
+        [Test]
+        public void InterruptUploadTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
         }
 
         private void PerformOneClientTest(ICodexNode primary)

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -22,26 +22,6 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloadedFile);
         }
 
-        [Test]
-        public void FindBug()
-        {
-            var uploader = StartCodex();
-            var downloaders = StartCodex(10);
-
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(15))
-            {
-                var cid = uploader.UploadFile(GenerateTestFile(5.MB()));
-
-                var loop = Parallel.ForEach(downloaders, d =>
-                {
-                    d.DownloadContent(cid);
-                });
-
-                Assert.That(loop.IsCompleted);
-            }
-        }
-
         [Test]
         public void DownloadingUnknownCidDoesNotCauseCrash()
         {

Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs

@@ -0,0 +1,228 @@
+using CodexPlugin;
+using FileUtils;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class MultiswarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        public void Multiswarm(
+            [Values(3, 5)] int numFiles,
+            [Values(5, 20)] int fileSizeMb,
+            [Values(1)] int uploadersPerFile,
+            [Values(3)] int downloadersPerFile,
+            [Values(1)] int maxUploadsPerNode,
+            [Values(2, 3)] int maxDownloadsPerNode
+        )
+        {
+            var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            Assert.That(plan.NodePlans.Count, Is.LessThan(30));
+
+            RunThePlan(plan, fileSizeMb);
+        }
+
+        private void RunThePlan(Plan plan, int fileSizeMb)
+        {
+            foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB());
+            var nodes = StartCodex(plan.NodePlans.Count);
+            for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i];
+
+            // Upload all files to their nodes.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var uploader in filePlan.Uploaders)
+                {
+                    filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!);
+                }
+            }
+
+            Thread.Sleep(5000); // Everything is processed and announced.
+
+            // Start all downloads (almost) simultaneously.
+            var tasks = new List<Task>();
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloader in filePlan.Downloaders)
+                {
+                    tasks.Add(Task.Run(() =>
+                    {
+                        var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!);
+                        lock (filePlan.DownloadedFiles)
+                        {
+                            filePlan.DownloadedFiles.Add(downloadedFile);
+                        }
+                    }));
+                }
+            }
+
+            Task.WaitAll(tasks.ToArray());
+
+            // Assert all files are correct.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloadedFile in filePlan.DownloadedFiles)
+                {
+                    filePlan.File!.AssertIsEqual(downloadedFile);
+                }
+            }
+        }
+
+        private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            plan.Initialize();
+            plan.LogPlan(GetTestLog());
+            return plan;
+        }
+    }
+
+    public class FilePlan
+    {
+        public FilePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public TrackedFile? File { get; set; }
+        public ContentId? Cid { get; set; }
+        public List<TrackedFile?> DownloadedFiles { get; } = new List<TrackedFile?>();
+        public List<NodePlan> Uploaders { get; } = new List<NodePlan>();
+        public List<NodePlan> Downloaders { get; } = new List<NodePlan>();
+
+        public override string ToString()
+        {
+            return $"FilePlan[{Number}] " +
+                $"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " +
+                $"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class NodePlan
+    {
+        public NodePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public ICodexNode? Node { get; set; }
+        public List<FilePlan> Uploads { get; } = new List<FilePlan>();
+        public List<FilePlan> Downloads { get; } = new List<FilePlan>();
+
+        public bool Contains(FilePlan plan)
+        {
+            return Uploads.Contains(plan) || Downloads.Contains(plan);
+        }
+
+        public override string ToString()
+        {
+            return $"NodePlan[{Number}] " +
+                $"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " +
+                $"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class Plan
+    {
+        private readonly int numFiles;
+        private readonly int uploadersPerFile;
+        private readonly int downloadersPerFile;
+        private readonly int maxUploadsPerNode;
+        private readonly int maxDownloadsPerNode;
+
+        public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            this.numFiles = numFiles;
+            this.uploadersPerFile = uploadersPerFile;
+            this.downloadersPerFile = downloadersPerFile;
+            this.maxUploadsPerNode = maxUploadsPerNode;
+            this.maxDownloadsPerNode = maxDownloadsPerNode;
+        }
+
+        public List<FilePlan> FilePlans { get; } = new List<FilePlan>();
+        public List<NodePlan> NodePlans { get; } = new List<NodePlan>();
+
+        public void Initialize()
+        {
+            for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i));
+            foreach (var filePlan in FilePlans)
+            {
+                while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan);
+                while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan);
+            }
+
+            CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number));
+            CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number));
+
+            foreach (var filePlan in FilePlans)
+            {
+                Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile));
+                Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile));
+            }
+            foreach (var nodePlan in NodePlans)
+            {
+                Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode));
+                Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode));
+            }
+        }
+
+        public void LogPlan(ILog log)
+        {
+            log.Log("The plan:");
+            log.Log("Input:");
+            log.Log($"numFiles: {numFiles}");
+            log.Log($"uploadersPerFile: {uploadersPerFile}");
+            log.Log($"downloadersPerFile: {downloadersPerFile}");
+            log.Log($"maxUploadsPerNode: {maxUploadsPerNode}");
+            log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}");
+            log.Log("Setup:");
+            log.Log($"number of nodes: {NodePlans.Count}");
+            foreach (var filePlan in FilePlans) log.Log(filePlan.ToString());
+            foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString());
+        }
+
+        private void AddDownloader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateDownloaderNode(filePlan);
+            filePlan.Downloaders.Add(nodePlan);
+            nodePlan.Downloads.Add(filePlan);
+        }
+
+        private void AddUploader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateUploaderNode(filePlan);
+            filePlan.Uploaders.Add(nodePlan);
+            nodePlan.Uploads.Add(filePlan);
+        }
+
+        private NodePlan GetOrCreateDownloaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+
+        private NodePlan GetOrCreateUploaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+    }
+}

Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs

@@ -0,0 +1,25 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class SwarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [CreateTranscript("swarm_retransmit")]
+        public void DetectBlockRetransmits(
+            [Values(1, 5, 10, 20)] int fileSize,
+            [Values(3, 5, 10, 20)] int numNodes
+        )
+        {
+            var nodes = StartCodex(numNodes);
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = nodes[0].UploadFile(file);
+
+            var tasks = nodes.Select(n => Task.Run(() => n.DownloadContent(cid))).ToArray();
+            Task.WaitAll(tasks);
+        }
+    }
+}

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests
         private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
         private readonly List<ulong> rewardsSeen = new List<ulong>();
         private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
-        private readonly List<string> receivedEvents = new List<string>();
+        private readonly List<ChainEventMessage> receivedEvents = new List<ChainEventMessage>();
  
         [Test]
         [DontDownloadLogs]
@@ -73,13 +73,18 @@ namespace CodexTests.UtilityTests
 
         private void AssertEventOccurance(string msg, int expectedCount)
         {
-            Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
+            Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount),
                 $"Event '{msg}' did not occure correct number of times.");
         }
 
         private void OnCommand(string timestamp, GiveRewardsCommand call)
         {
             Log($"<API call {timestamp}>");
+            foreach (var e in call.EventsOverview)
+            {
+                Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order.");
+            }
+
             receivedEvents.AddRange(call.EventsOverview);
             foreach (var e in call.EventsOverview)
             {

Tests/DistTestCore/Configuration.cs

@@ -29,12 +29,12 @@ namespace DistTestCore
         /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace, Func<string?, string?> replacer)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
         {
-            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace, replacer);
+            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
         }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace, Func<string?, string?> replacer)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
         {
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
@@ -45,7 +45,6 @@ namespace DistTestCore
 
             config.AllowNamespaceOverride = false;
             config.Hooks = hooks;
-            config.Replacer = replacer;
 
             return config;
         }

Tests/DistTestCore/DistTest.cs

@@ -35,7 +35,7 @@ namespace DistTestCore
             fixtureLog = new FixtureLog(logConfig, startTime, deployId);
             statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix, s => s), configuration.GetFileManagerFolder());
+            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
 
             Initialize(fixtureLog);
         }

Tests/DistTestCore/TestLifecycle.cs

@@ -25,7 +25,7 @@ namespace DistTestCore
             TestNamespace = testNamespace;
             TestStart = DateTime.UtcNow;
 
-            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace, InternalReplacer), configuration.GetFileManagerFolder(), timeSet);
+            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
             this.deployId = deployId;
@@ -33,11 +33,6 @@ namespace DistTestCore
             log.WriteLogTag();
         }
 
-        private string? InternalReplacer(string? arg)
-        {
-            return Replacer(arg);
-        }
-
         public DateTime TestStart { get; }
         public TestLog Log { get; }
         public Configuration Configuration { get; }
@@ -45,7 +40,6 @@ namespace DistTestCore
         public string TestNamespace { get; }
         public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
-        public Func<string?, string?> Replacer { get; set; } = s => s;
 
         public void DeleteAllResources()
         {

Tests/FrameworkTests/FrameworkTests.csproj

@@ -18,6 +18,7 @@
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
   </ItemGroup>
 
 </Project>

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -19,11 +19,20 @@ namespace FrameworkTests.NethereumWorkflow
         {
             var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30);
             blocks = new Dictionary<ulong, Block>();
-            
+
+            Block? prev = null;
             for (ulong i = 0; i < 30; i++)
             {
                 ulong d = 100 + i;
-                blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2)));
+                var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2));
+                blocks.Add(d, newBlock);
+
+                if (prev != null)
+                {
+                    prev.Next = newBlock;
+                    newBlock.Previous = prev;
+                }
+                prev = newBlock;
             }
         }
 
@@ -99,23 +108,23 @@ namespace FrameworkTests.NethereumWorkflow
         }
 
         [Test]
-        public void FailsToFindBlockBeforeFrontOfChain()
+        public void FindsGenesisBlockAtFrontOfChain()
         {
             var first = blocks.First().Value;
 
-            var notFound = finder.GetHighestBlockNumberBefore(first.Time);
+            var firstNumber = finder.GetHighestBlockNumberBefore(first.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(firstNumber, Is.EqualTo(first.Number));
         }
 
         [Test]
-        public void FailsToFindBlockAfterTailOfChain()
+        public void FindsCurrentBlockAtTailOfChain()
         {
             var last = blocks.Last().Value;
 
-            var notFound = finder.GetLowestBlockNumberAfter(last.Time);
+            var lastNumber = finder.GetLowestBlockNumberAfter(last.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(lastNumber, Is.EqualTo(last.Number));
         }
 
         [Test]
@@ -143,13 +152,27 @@ namespace FrameworkTests.NethereumWorkflow
         {
             foreach (var pair in blocks)
             {
-                finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
-                finder.GetHighestBlockNumberBefore(pair.Value.Time);
-                finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
+                var block = pair.Value;
 
-                finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
-                finder.GetLowestBlockNumberAfter(pair.Value.Time);
-                finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
+                AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter));
+
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore));
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time));
+                AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter));
+            }
+        }
+
+        private void AssertLink(Block? expected, ulong? actual)
+        {
+            if (expected == null)
+            {
+                Assert.That(actual, Is.Null);
+            }
+            else
+            {
+                Assert.That(expected.Number, Is.EqualTo(actual!.Value));
             }
         }
     }
@@ -167,6 +190,9 @@ namespace FrameworkTests.NethereumWorkflow
         public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
         public DateTime JustAfter { get { return Time.AddSeconds(1); } }
 
+        public Block? Next { get; set; }
+        public Block? Previous { get; set; }
+
         public override string ToString()
         {
             return $"[{Number}]";

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -1,136 +0,0 @@
-using Logging;
-using NUnit.Framework;
-using OverwatchTranscript;
-
-namespace FrameworkTests.OverwatchTranscript
-{
-    [TestFixture]
-    public class TranscriptTests
-    {
-        private const string TranscriptFilename = "testtranscript.owts";
-        private const string HeaderKey = "testHeader";
-        private const string HeaderData = "abcdef";
-        private const string EventData0 = "12345";
-        private const string EventData1 = "678";
-        private const string EventData2 = "90";
-        private const string EventData3 = "-=";
-        private readonly DateTime t0 = DateTime.UtcNow;
-        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
-        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
-
-        [Test]
-        public void WriteAndRun()
-        {
-            // unstable.
-            WriteTranscript();
-            ReadTranscript();
-
-            File.Delete(TranscriptFilename);
-        }
-
-        private void WriteTranscript()
-        {
-            var log = new ConsoleLog();
-            var writer = Transcript.NewWriter(log);
-
-            writer.AddHeader(HeaderKey, new TestHeader
-            {
-                HeaderData = HeaderData
-            });
-
-            writer.Add(t0, new MyEvent
-            {
-                EventData = EventData0
-            });
-            writer.Add(t2, new MyEvent
-            {
-                EventData = EventData3
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData1
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData2
-            });
-
-            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
-
-            writer.Write(TranscriptFilename);
-        }
-
-        private void ReadTranscript()
-        {
-            var reader = Transcript.NewReader(TranscriptFilename);
-
-            var header = reader.GetHeader<TestHeader>(HeaderKey);
-            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
-            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
-            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
-            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
-            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
-
-            var moments = new List<ActivateMoment>();
-            var events = new List<ActivateEvent<MyEvent>>();
-            reader.AddMomentHandler(moments.Add);
-            reader.AddEventHandler<MyEvent>(events.Add);
-
-
-            Assert.That(moments.Count, Is.EqualTo(0));
-            Assert.That(events.Count, Is.EqualTo(0));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(1));
-            Assert.That(events.Count, Is.EqualTo(1));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(2));
-            Assert.That(events.Count, Is.EqualTo(3));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
-            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
-            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
-
-            AssertEvent(events[0], utc: t0, duration: t1 - t0, index: 0, data: EventData0);
-            AssertEvent(events[1], utc: t1, duration: t2 - t1, index: 1, data: EventData1);
-            AssertEvent(events[2], utc: t1, duration: t2 - t1, index: 1, data: EventData2);
-            AssertEvent(events[3], utc: t2, duration: null, index: 2, data: EventData3);
-
-            reader.Close();
-        }
-
-        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
-        {
-            Assert.That(m.Utc, Is.EqualTo(utc));
-            Assert.That(m.Duration, Is.EqualTo(duration));
-            Assert.That(m.Index, Is.EqualTo(index));
-        }
-
-        private void AssertEvent(ActivateEvent<MyEvent> e, DateTime utc, TimeSpan? duration, int index, string data)
-        {
-            Assert.That(e.Moment.Utc, Is.EqualTo(utc));
-            Assert.That(e.Moment.Duration, Is.EqualTo(duration));
-            Assert.That(e.Moment.Index, Is.EqualTo(index));
-            Assert.That(e.Payload.EventData, Is.EqualTo(data));
-        }
-    }
-
-    public class TestHeader
-    {
-        public string HeaderData { get; set; } = string.Empty;
-    }
-
-    public class MyEvent
-    {
-        public string EventData { get; set; } = string.Empty;
-    }
-}

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptLargeTest.cs

@@ -2,7 +2,7 @@
 using NUnit.Framework;
 using OverwatchTranscript;
 
-namespace FrameworkTests.OverwatchTranscript
+namespace FrameworkTests.OverwatchTranscriptTests
 {
     [TestFixture]
     public class TranscriptLargeTests

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptTests.cs

@@ -0,0 +1,229 @@
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using OverwatchTranscript;
+using System.IO.Compression;
+
+namespace FrameworkTests.OverwatchTranscriptTests
+{
+    [TestFixture]
+    public class TranscriptTests
+    {
+        private const string TranscriptFilename = "testtranscript.owts";
+        private const string HeaderKey = "testHeader";
+        private const string HeaderData = "abcdef";
+        private const string EventData0 = "12345";
+        private const string EventData1 = "678";
+        private const string EventData2 = "90";
+        private const string EventData3 = "-=";
+        private readonly DateTime t0 = DateTime.UtcNow;
+        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
+        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
+
+        [Test]
+        [Combinatorial]
+        public void WriteAndRun()
+        {
+            WriteTranscript();
+            AssertFileContent();
+            ReadTranscript();
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private void WriteTranscript()
+        {
+            var log = new ConsoleLog();
+            var writer = Transcript.NewWriter(log);
+
+            writer.AddHeader(HeaderKey, new TestHeader
+            {
+                HeaderData = HeaderData
+            });
+
+            writer.Add(t0, new MyEvent
+            {
+                EventData = EventData0
+            });
+            writer.Add(t2, new MyEvent
+            {
+                EventData = EventData3
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData1
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData2
+            });
+
+            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
+
+            writer.Write(TranscriptFilename);
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var header = reader.GetHeader<TestHeader>(HeaderKey);
+            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
+            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
+
+            var moments = new List<ActivateMoment>();
+            var events = new List<ActivateEvent<MyEvent>>();
+            reader.AddMomentHandler(moments.Add);
+            reader.AddEventHandler<MyEvent>(events.Add);
+
+            var timeout = 10;
+            while (moments.Count < 3 && events.Count < 4)
+            {
+                Thread.Sleep(10);
+                reader.Next();
+
+                timeout--;
+                if (timeout == 0) Assert.Fail("Events not received.");
+            }
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
+            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
+            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
+
+            AssertEvent(events, utc: t0, duration: t1 - t0, index: 0, data: EventData0);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData2);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData1);
+            AssertEvent(events, utc: t2, duration: null, index: 2, data: EventData3);
+
+            reader.Close();
+        }
+
+        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
+        {
+            Assert.That(m.Utc, Is.EqualTo(utc));
+            Assert.That(m.Duration, Is.EqualTo(duration));
+            Assert.That(m.Index, Is.EqualTo(index));
+        }
+
+        private void AssertEvent(List<ActivateEvent<MyEvent>> events, DateTime utc, TimeSpan? duration, int index, string data)
+        {
+            var e = events.SingleOrDefault(e => e.Moment.Utc == utc && e.Payload.EventData == data);
+            if (e == null) Assert.Fail("Event not found");
+
+            Assert.That(e!.Moment.Utc, Is.EqualTo(utc));
+            Assert.That(e!.Moment.Duration, Is.EqualTo(duration));
+            Assert.That(e!.Moment.Index, Is.EqualTo(index));
+            Assert.That(e!.Payload.EventData, Is.EqualTo(data));
+        }
+
+        private void AssertFileContent()
+        {
+            using var zip = ZipFile.OpenRead(TranscriptFilename);
+            Assert.That(zip.Entries.Count, Is.EqualTo(2));
+            foreach (var entry in zip.Entries)
+            {
+                if (entry.Name == "transcript.json")
+                {
+                    var transcript = ZipEntryJson<OverwatchTranscript.OverwatchTranscript>(entry);
+                    AssertTranscript(transcript);
+                }
+                else
+                {
+                    var moments = ZipEntryToMoments(entry);
+                    AssertMoments(moments);
+                }
+            }
+        }
+
+        private void AssertTranscript(OverwatchTranscript.OverwatchTranscript transcript)
+        {
+            Assert.That(transcript.Header.Common.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.Header.Common.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.Header.Common.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.Header.Common.LatestUtc, Is.EqualTo(t2));
+            Assert.That(transcript.Header.Entries.Length, Is.EqualTo(1));
+            Assert.That(transcript.Header.Entries[0].Key, Is.EqualTo(HeaderKey));
+            Assert.That(transcript.Header.Entries[0].Value, Is.EqualTo("{\"HeaderData\":\"abcdef\"}"));
+
+            Assert.That(transcript.MomentReferences.Length, Is.EqualTo(1));
+            Assert.That(transcript.MomentReferences[0].NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.MomentReferences[0].NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.MomentReferences[0].EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.MomentReferences[0].LatestUtc, Is.EqualTo(t2));
+        }
+
+        private void AssertMoments(OverwatchMoment[] moments)
+        {
+            Assert.That(moments.Length, Is.EqualTo(3));
+
+            Assert.That(moments[0].Utc, Is.EqualTo(t0));
+            Assert.That(moments[0].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[0].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[0].Events[0].Payload, Is.EqualTo("{\"EventData\":\"12345\"}"));
+
+            Assert.That(moments[1].Utc, Is.EqualTo(t1));
+            Assert.That(moments[1].Events.Length, Is.EqualTo(2));
+            Assert.That(moments[1].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[1].Events[1].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+
+            // output order is not guaranteed:
+            var payloads = moments[1].Events.Select(e => e.Payload).ToArray();
+            CollectionAssert.AreEquivalent(new[]
+            {
+                "{\"EventData\":\"90\"}",
+                "{\"EventData\":\"678\"}"
+            }, payloads);
+
+            Assert.That(moments[2].Utc, Is.EqualTo(t2));
+            Assert.That(moments[2].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[2].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[2].Events[0].Payload, Is.EqualTo("{\"EventData\":\"-=\"}"));
+        }
+
+        private T ZipEntryJson<T>(ZipArchiveEntry? entry)
+        {
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+            var json = reader.ReadToEnd();
+            var result = JsonConvert.DeserializeObject<T>(json);
+            if (result == null) Assert.Fail("didn't deserialize");
+            return result!;
+        }
+
+        private OverwatchMoment[] ZipEntryToMoments(ZipArchiveEntry? entry)
+        {
+            var result = new List<OverwatchMoment>();
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+
+            var line = reader.ReadLine();
+            while (!string.IsNullOrEmpty(line))
+            {
+                var moment = JsonConvert.DeserializeObject<OverwatchMoment>(line);
+                if (moment != null) result.Add(moment);
+                line = reader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+    }
+
+    public class TestHeader
+    {
+        public string HeaderData { get; set; } = string.Empty;
+    }
+
+    public class MyEvent
+    {
+        public string EventData { get; set; } = string.Empty;
+    }
+}

Tests/FrameworkTests/Utils/EmojiMapsTests.cs

@@ -0,0 +1,28 @@
+using NUnit.Framework;
+using System.Text;
+using TestNetRewarder;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EmojiMapsTests
+    {
+        private readonly Random random = new Random();
+        private readonly EmojiMaps maps = new EmojiMaps();
+
+        [Test]
+        public void GeneratesConsistentStrings(
+            [Values(1, 5, 10, 20)] int inputLength,
+            [Values(1, 2, 3, 5)] int outLength)
+        {
+            var buffer = new byte[inputLength];
+            random.NextBytes(buffer);
+            var input = Encoding.ASCII.GetString(buffer);
+
+            var out1 = maps.StringToEmojis(input, outLength);
+            var out2 = maps.StringToEmojis(input, outLength);
+            
+            Assert.That(out1, Is.EqualTo(out2));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncoding.cs

@@ -0,0 +1,334 @@
+namespace FrameworkTests.Utils
+{
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public RunUpdate ExpandToInclude(int index)
+        {
+            if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}");
+            if (index == (Start + Length))
+            {
+                Length++;
+                return new RunUpdate();
+            }
+            if (index == (Start - 1))
+            {
+                return new RunUpdate(
+                    newRuns: [new Run(Start - 1, Length + 1)],
+                    removeRuns: [this]
+                );
+            }
+            throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}");
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(
+                        newRuns: Array.Empty<Run>(),
+                        removeRuns: [this]
+                    );
+                }
+                return new RunUpdate(
+                    newRuns: [new Run(Start + 1, Length - 1)],
+                    removeRuns: [this]
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(
+                newRuns: [new Run(index + 1, newRunLength)],
+                removeRuns: Array.Empty<Run>()
+            );
+        }
+
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+
+        public override string ToString()
+        {
+            return $"[{Start},{Length}]";
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is Run run &&
+                   Start == run.Start &&
+                   Length == run.Length;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Start, Length);
+        }
+
+        public static bool operator ==(Run? obj1, Run? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2);
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+
+    public partial class IndexSet
+    {
+        private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+        public IndexSet()
+        {
+        }
+
+        public IndexSet(int[] indices)
+        {
+            foreach (var i in indices) Set(i);
+        }
+
+        public static IndexSet FromRunLengthEncoded(int[] rle)
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < rle.Length; i += 2)
+            {
+                var start = rle[i];
+                var length = rle[i + 1];
+                set.runs.Add(start, new Run(start, length));
+            }
+
+            return set;
+        }
+
+        public bool IsSet(int index)
+        {
+            if (runs.ContainsKey(index)) return true;
+
+            var run = GetRunAt(index);
+            if (run == null) return false;
+            return true;
+        }
+
+        public void Set(int index)
+        {
+            if (IsSet(index)) return;
+
+            var runBefore = GetRunAt(index - 1);
+            var runAfter = GetRunExact(index + 1);
+
+            if (runBefore == null)
+            {
+                if (runAfter == null)
+                {
+                    CreateNewRun(index);
+                }
+                else
+                {
+                    HandleUpdate(runAfter.ExpandToInclude(index));
+                }
+            }
+            else
+            {
+                if (runAfter == null)
+                {
+                    HandleUpdate(runBefore.ExpandToInclude(index));
+                }
+                else
+                {
+                    // new index will connect runBefore with runAfter. We merge!
+                    HandleUpdate(new RunUpdate(
+                        newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)],
+                        removeRuns: [runBefore, runAfter]
+                    ));
+                }
+            }
+        }
+
+        public void Unset(int index)
+        {
+            if (runs.ContainsKey(index))
+            {
+                HandleUpdate(runs[index].Unset(index));
+            }
+            else
+            {
+                var run = GetRunAt(index);
+                if (run == null) return;
+                HandleUpdate(run.Unset(index));
+            }
+        }
+
+        public void Iterate(Action<int> onIndex)
+        {
+            foreach (var run in runs.Values)
+            {
+                run.Iterate(onIndex);
+            }
+        }
+
+        public int[] RunLengthEncoded()
+        {
+            return Encode().ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join("&", runs.Select(r => r.ToString()).ToArray());
+        }
+
+        private IEnumerable<int> Encode()
+        {
+            foreach (var pair in runs)
+            {
+                yield return pair.Value.Start;
+                yield return pair.Value.Length;
+            }
+        }
+
+        private Run? GetRunAt(int index)
+        {
+            foreach (var run in runs.Values)
+            {
+                if (run.Includes(index)) return run;
+            }
+            return null;
+        }
+
+        private Run? GetRunExact(int index)
+        {
+            if (runs.ContainsKey(index)) return runs[index];
+            return null;
+        }
+
+        private void HandleUpdate(RunUpdate runUpdate)
+        {
+            foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+        }
+
+        private void CreateNewRun(int index)
+        {
+            if (runs.ContainsKey(index + 1))
+            {
+                var length = runs[index + 1].Length + 1;
+                runs.Add(index, new Run(index, length));
+                runs.Remove(index + 1);
+            }
+            else
+            {
+                runs.Add(index, new Run(index, 1));
+            }
+        }
+    }
+
+    public partial class IndexSet
+    {
+        public IndexSet Overlap(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public IndexSet Merge(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(result.Set);
+            other.Iterate(result.Set);
+            return result;
+        }
+
+        public IndexSet Without(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (!other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (obj is IndexSet set)
+            {
+                if (set.runs.Count != runs.Count) return false;
+                foreach (var pair in runs)
+                {
+                    if (!set.runs.ContainsKey(pair.Key)) return false;
+                    if (set.runs[pair.Key] != pair.Value) return false;
+                }
+                return true;
+            }
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(runs);
+        }
+
+        public static bool operator ==(IndexSet? obj1, IndexSet? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2);
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs

@@ -0,0 +1,94 @@
+using NUnit.Framework;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingLogicalTests
+    {
+        [Test]
+        public void EqualityTest()
+        {
+            var setA = new IndexSet([1, 2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.EqualTo(setB));
+            Assert.That(setA == setB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var setA = new IndexSet([1, 2, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var setA = new IndexSet([1, 2, 3]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest3()
+        {
+            var setA = new IndexSet([2, 3, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest()
+        {
+            var setA = new IndexSet([2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void Overlap()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([3, 4, 5, 11]);
+
+            var set = setA.Overlap(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Merge()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]);
+
+            var set = setA.Merge(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Without()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 14]);
+
+            var set = setA.Without(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -1,12 +1,40 @@
 using NUnit.Framework;
-using NUnit.Framework.Interfaces;
-using static FrameworkTests.Utils.RunLengthEncodingTests;
 
 namespace FrameworkTests.Utils
 {
     [TestFixture]
     public class RunLengthEncodingRunTests
     {
+        [Test]
+        public void EqualityTest()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 4);
+
+            Assert.That(runA, Is.EqualTo(runB));
+            Assert.That(runA == runB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 5);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(2, 4);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
         [Test]
         [Combinatorial]
         public void RunIncludes(
@@ -33,23 +61,58 @@ namespace FrameworkTests.Utils
         }
 
         [Test]
-        public void RunExpandToInclude()
+        public void RunExpandThrowsWhenIndexNotAdjacent()
         {
             var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
             Assert.That(run.Includes(2));
             Assert.That(run.Includes(4));
             Assert.That(!run.Includes(5));
 
-            Assert.That(run.ExpandToInclude(1), Is.False);
-            Assert.That(run.ExpandToInclude(2), Is.False);
-            Assert.That(run.ExpandToInclude(4), Is.False);
-            Assert.That(run.ExpandToInclude(6), Is.False);
+            Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf<Exception>());
+        }
 
-            Assert.That(run.ExpandToInclude(5), Is.True);
+        [Test]
+        public void RunExpandThrowsWhenIndexAlreadyIncluded()
+        {
+            var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf<Exception>());
+        }
+
+        [Test]
+        public void RunExpandToIncludeAfter()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(5);
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
             Assert.That(run.Includes(5));
             Assert.That(!run.Includes(6));
         }
 
+        [Test]
+        public void RunExpandToIncludeBefore()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(1);
+
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
         [Test]
         public void RunCanUnsetLastIndex()
         {
@@ -100,94 +163,9 @@ namespace FrameworkTests.Utils
         {
             var run = new Run(2, 4);
             var seen = new List<int>();
-            run.Iterate(i => seen.Add(i));
+            run.Iterate(seen.Add);
 
             CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
         }
     }
-
-    public class Run
-    {
-        public Run(int start, int length)
-        {
-            Start = start;
-            Length = length;
-        }
-
-        public int Start { get; }
-        public int Length { get; private set; }
-
-        public bool Includes(int index)
-        {
-            return index >= Start && index < (Start + Length);
-        }
-
-        public bool ExpandToInclude(int index)
-        {
-            if (index == (Start + Length))
-            {
-                Length++;
-                return true;
-            }
-            return false;
-        }
-
-        public RunUpdate Unset(int index)
-        {
-            if (!Includes(index))
-            {
-                return new RunUpdate();
-            }
-
-            if (index == Start)
-            {
-                // First index: Replace self with new run at next index, unless empty.
-                if (Length == 1)
-                {
-                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
-                }
-                return new RunUpdate(
-                    newRuns: new[] { new Run(Start + 1, Length - 1) },
-                    removeRuns: new[] { this }
-                );
-            }
-
-            if (index == (Start + Length - 1))
-            {
-                // Last index: Become one smaller.
-                Length--;
-                return new RunUpdate();
-            }
-
-            // Split:
-            var newRunLength = (Start + Length - 1) - index;
-            Length = index - Start;
-            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
-        }
-        
-        public void Iterate(Action<int> action)
-        {
-            for (var i = 0; i < Length; i++)
-            {
-                action(Start + i);
-            }
-        }
-    }
-
-    public class RunUpdate
-    {
-        public RunUpdate()
-            : this(Array.Empty<Run>(), Array.Empty<Run>())
-        {
-        }
-
-        public RunUpdate(Run[] newRuns, Run[] removeRuns)
-        {
-            NewRuns = newRuns;
-            RemoveRuns = removeRuns;
-        }
-
-        public Run[] NewRuns { get; }
-        public Run[] RemoveRuns { get; }
-    }
 }

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -1,9 +1,4 @@
-using Logging;
-using Microsoft.VisualStudio.TestPlatform.Common;
-using NuGet.Frameworks;
-using NUnit.Framework;
-using System.Collections.Concurrent;
-using System.Numerics;
+using NUnit.Framework;
 using Utils;
 
 namespace FrameworkTests.Utils
@@ -119,6 +114,19 @@ namespace FrameworkTests.Utils
             }, encoded);
         }
 
+        [Test]
+        public void SetIndexBetweenRuns()
+        {
+            var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                8, 7
+            }, encoded);
+        }
+
         [Test]
         public void SetIndexAfterRun()
         {
@@ -201,120 +209,5 @@ namespace FrameworkTests.Utils
             all.Sort();
             return all.ToArray();
         }
-
-        public class IndexSet
-        {
-            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
-
-            public IndexSet()
-            {
-            }
-
-            public IndexSet(int[] indices)
-            {
-                foreach (var i in indices) Set(i);
-            }
-
-            public static IndexSet FromRunLengthEncoded(int[] rle)
-            {
-                var set = new IndexSet();
-                for (var i = 0; i < rle.Length; i += 2)
-                {
-                    var start = rle[i];
-                    var length = rle[i + 1];
-                    set.runs.Add(start, new Run(start, length));
-                }
-
-                return set;
-            }
-
-            public bool IsSet(int index)
-            {
-                if (runs.ContainsKey(index)) return true;
-
-                var run = GetRunBefore(index);
-                if (run == null) return false;
-
-                return run.Includes(index);
-            }
-
-            public void Set(int index)
-            {
-                if (runs.ContainsKey(index)) return;
-
-                var run = GetRunBefore(index);
-                if (run == null || !run.ExpandToInclude(index))
-                {
-                    CreateNewRun(index);
-                }
-            }
-
-            public void Unset(int index)
-            {
-                if (runs.ContainsKey(index))
-                {
-                    HandleUpdate(runs[index].Unset(index));
-                }
-                else
-                {
-                    var run = GetRunBefore(index);
-                    if (run == null) return;
-                    HandleUpdate(run.Unset(index));
-                }
-            }
-
-            public void Iterate(Action<int> onIndex)
-            {
-                foreach (var run in runs.Values)
-                {
-                    run.Iterate(onIndex);
-                }
-            }
-
-            public int[] RunLengthEncoded()
-            {
-                return Encode().ToArray();
-            }
-
-            private IEnumerable<int> Encode()
-            {
-                foreach (var pair in runs)
-                {
-                    yield return pair.Value.Start;
-                    yield return pair.Value.Length;
-                }
-            }
-
-            private Run? GetRunBefore(int index)
-            {
-                Run? result = null;
-                foreach (var pair in runs)
-                {
-                    if (pair.Key < index) result = pair.Value;
-                    else return result;
-                }
-                return result;
-            }
-
-            private void HandleUpdate(RunUpdate runUpdate)
-            {
-                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
-                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
-            }
-
-            private void CreateNewRun(int index)
-            {
-                if (runs.ContainsKey(index + 1))
-                {
-                    var length = runs[index + 1].Length + 1;
-                    runs.Add(index, new Run(index, length));
-                    runs.Remove(index + 1);
-                }
-                else
-                {
-                    runs.Add(index, new Run(index, 1));
-                }
-            }
-        }
     }
 }

Tools/AutoClient/App.cs

@@ -1,4 +1,5 @@
-using Logging;
+using AutoClient.Modes.FolderStore;
+using Logging;
 
 namespace AutoClient
 {
@@ -19,6 +20,15 @@ namespace AutoClient
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
+
+            if (!string.IsNullOrEmpty(config.FolderToStore))
+            {
+                FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
+            }
+            else
+            {
+                FolderWorkDispatcher = null!;
+            }
         }
 
         public Configuration Config { get; }
@@ -27,6 +37,7 @@ namespace AutoClient
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public CidRepo CidRepo { get; }
         public Performance Performance { get; }
+        public FolderWorkDispatcher FolderWorkDispatcher { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -0,0 +1,165 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class AutomaticPurchaser
+    {
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly CodexNode codex;
+        private Task workerTask = Task.CompletedTask;
+        private App app => instance.App;
+
+        public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
+        {
+            this.log = log;
+            this.instance = instance;
+            this.codex = codex;
+        }
+
+        public void Start()
+        {
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
+        }
+
+        private async Task Worker()
+        {
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(instance.NodeId);
+            if (cid == null) return;
+
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+            await codex.DownloadCid(filename, cid, size);
+
+            DeleteFile(filename);
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            try
+            {
+                var cid = await codex.UploadFile(file);
+                var response = await codex.RequestStorage(cid);
+                return response.PurchaseId;
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (!app.Cts.Token.IsCancellationRequested)
+                {
+                    var purchase = await codex.GetStoragePurchase(pid);
+                    if (purchase == null)
+                    {
+                        await FixedShortDelay();
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                        continue;
+                    }
+                    if (purchase.IsCancelled)
+                    {
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (purchase.IsError)
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (purchase.IsFinished)
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (purchase.IsStarted)
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, app.Cts.Token);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -17,6 +17,7 @@
             lock (_lock)
             {
                 entries.Add(new CidEntry(nodeId, cid, knownSize));
+                if (entries.Count > 1000) entries.Clear();
             }
         }
 

Tools/AutoClient/CodexInstance.cs

@@ -0,0 +1,161 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Nethereum.Model;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public interface ICodexInstance
+    {
+        string NodeId { get; }
+        App App { get; }
+        CodexApi Codex { get; }
+        HttpClient Client { get; }
+        Address Address { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            App = app;
+            Codex = codex;
+            Client = client;
+            Address = address;
+            NodeId = Guid.NewGuid().ToString();
+        }
+
+        public string NodeId { get; }
+        public App App { get; }
+        public CodexApi Codex { get; }
+        public HttpClient Client { get; }
+        public Address Address { get; }
+    }
+
+    public class CodexNode
+    {
+        private readonly App app;
+        private readonly ICodexInstance codex;
+
+        public CodexNode(App app, ICodexInstance instance)
+        {
+            this.app = app;
+            codex = instance;
+        }
+
+        public async Task DownloadCid(string filename, string cid, long? size)
+        {
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                using var fileStream = File.OpenWrite(filename);
+                var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
+                fileResponse.Stream.CopyTo(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.DownloadSuccessful(size, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
+            }
+        }
+
+        public async Task<ContentId> UploadFile(string filename)
+        {
+            using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream, filename);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
+
+        public async Task<RequestStorageResult> RequestStorage(ContentId cid)
+        {
+            app.Log.Debug("Requesting storage for " + cid.Id);
+            var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
+
+            app.Log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
+
+            return new RequestStorageResult(result, new ContentId(encoded));
+        }
+
+        public class RequestStorageResult
+        {
+            public RequestStorageResult(string purchaseId, ContentId encodedCid)
+            {
+                PurchaseId = purchaseId;
+                EncodedCid = encodedCid;
+            }
+
+            public string PurchaseId { get; }
+            public ContentId EncodedCid { get; }
+
+            public override string ToString()
+            {
+                return $"{PurchaseId} (cid: {EncodedCid})";
+            }
+        }
+
+        public async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
+        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
+        {
+            app.Log.Debug($"Uploading file...");
+            var response = await codex.Codex.UploadAsync(
+                content_type: "application/octet-stream",
+                content_disposition: $"attachment; filename=\"{filename}\"",
+                fileStream, app.Cts.Token);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> GetEncodedCid(string pid)
+        {
+            try
+            {
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
+            }
+            catch (Exception ex)
+            {
+                app.Log.Error(ex.ToString());
+                throw;
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -5,7 +5,8 @@ namespace AutoClient
     public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
-        public string CodexEndpoints { get; set; } = "http://localhost:8080";
+        public string CodexEndpoints { get; set; } =
+            "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -14,19 +15,22 @@ namespace AutoClient
         public int NumConcurrentPurchases { get; set; } = 10;
 
         [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
-        public int ContractDurationMinutes { get; set; } = 60 * 6;
+        public int ContractDurationMinutes { get; set; } =
+            60 * 24 * 6; // 6 days
+            //60 * 6; 6 hours
+        // Cluster nodes configured for max 7-day storage.
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 15;
+        public int ContractExpiryMinutes { get; set; } = 60;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
-        public int NumHosts { get; set; } = 10;
+        public int NumHosts { get; set; } = 5;
 
         [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
-        public int HostTolerance { get; set; } = 5;
+        public int HostTolerance { get; set; } = 1;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
-        public int Price { get; set; } = 10;
+        public int Price { get; set; } = 1000;
 
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
@@ -34,6 +38,9 @@ namespace AutoClient
         [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
         public int FileSizeMb { get; set; } = 0;
 
+        [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
+        public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
+
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -0,0 +1,58 @@
+using static AutoClient.Modes.FolderStore.FileWorker;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileStatus : JsonBacked<WorkerStatus>
+    {
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
+            : base(app, folder, filePath + ".json")
+        {
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public bool IsBusy()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                !p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyRunning()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyFailed()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            var mostRecent = GetMostRecent();
+            if (mostRecent == null) return false;
+
+            return mostRecent.Expiry.HasValue;
+        }
+
+        protected WorkerPurchase? GetMostRecent()
+        {
+            if (!State.Purchases.Any()) return null;
+            var maxCreated = State.Purchases.Max(p => p.Created);
+            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -0,0 +1,293 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileWorker : FileStatus
+    {
+        private readonly App app;
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly string sourceFilename;
+        private readonly Action onFileUploaded;
+        private readonly Action onNewPurchase;
+        private readonly CodexNode codex;
+
+        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
+            : base(app, folder, fileIndex.File + ".json", purchaseInfo)
+        {
+            this.app = app;
+            log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
+            this.instance = instance;
+            this.purchaseInfo = purchaseInfo;
+            sourceFilename = fileIndex.File;
+            if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
+            this.onFileUploaded = onFileUploaded;
+            this.onNewPurchase = onNewPurchase;
+            codex = new CodexNode(app, instance);
+        }
+
+        public int FailureCounter => State.FailureCounter;
+
+        protected override void OnNewState(WorkerStatus newState)
+        {
+            newState.LastUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update()
+        {
+            try
+            {
+                if (IsCurrentlyRunning() && UpdatedRecently()) return;
+
+                Log($"Updating for '{sourceFilename}'...");
+                await EnsureRecentPurchase();
+                SaveState();
+                app.Log.Log("");
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Exception during fileworker update: " + exc);
+                State.Error = exc.ToString();
+                SaveState();
+                throw;
+            }
+        }
+
+        private bool UpdatedRecently()
+        {
+            var now = DateTime.UtcNow;
+            return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
+        }
+
+        private async Task<string> EnsureCid()
+        {
+            Log($"Checking CID...");
+
+            if (!string.IsNullOrEmpty(State.EncodedCid) &&
+                await DoesCidExistInNetwork(State.EncodedCid))
+            {
+                Log("Encoded-CID successfully found in the network.");
+                // TODO: Using the encoded CID currently would result in double-encoding of the dataset.
+                // See: https://github.com/codex-storage/nim-codex/issues/1005
+                // Always use the basic CID for now, even though we have to repeat the encoding.
+                // When using encoded CID works: return State.EncodedCid;
+            }
+
+            if (!string.IsNullOrEmpty(State.Cid) &&
+                await DoesCidExistInNetwork(State.Cid))
+            {
+                Log("Basic-CID successfully found in the network.");
+                return State.Cid;
+            }
+
+            if (string.IsNullOrEmpty(State.Cid))
+            {
+                Log("File was not previously uploaded.");
+            }
+
+            Log($"Uploading...");
+            var cid = await codex.UploadFile(sourceFilename);
+            onFileUploaded();
+            Log("Got Basic-CID: " + cid);
+            State.Cid = cid.Id;
+            SaveState();
+            return State.Cid;
+        }
+
+        private async Task<bool> DoesCidExistInNetwork(string cid)
+        {
+            try
+            {
+                // This should not take longer than a few seconds. If it does, cancel it.
+                var cts = new CancellationTokenSource();
+                var cancelTask = Task.Run(() =>
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(15));
+                    cts.Cancel();
+                });
+
+                var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
+                if (manifest == null) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+        private async Task EnsureRecentPurchase()
+        {
+            Log($"Checking recent purchase...");
+            var recent = GetMostRecent();
+            if (recent == null)
+            {
+                Log($"No recent purchase.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            await UpdatePurchase(recent);
+
+            if (recent.Expiry.HasValue)
+            {
+                Log($"Purchase has failed or expired.");
+                await MakeNewPurchase();
+                State.FailureCounter++;
+                return;
+            }
+
+            if (recent.Finish.HasValue)
+            {
+                Log($"Purchase has finished.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
+            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
+            {
+                Log($"Purchase is going to expire soon.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            if (!recent.Submitted.HasValue)
+            {
+                Log($"Purchase is waiting to be submitted.");
+                return;
+            }
+
+            if (recent.Submitted.HasValue && !recent.Started.HasValue)
+            {
+                Log($"Purchase is submitted and waiting to start.");
+                return;
+            }
+
+            Log($"Purchase is running.");
+        }
+
+        private async Task UpdatePurchase(WorkerPurchase recent)
+        {
+            if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
+            var now = DateTime.UtcNow;
+
+            var purchase = await codex.GetStoragePurchase(recent.Pid);
+            if (purchase == null)
+            {
+                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
+                recent.Expiry = now;
+                return;
+            }
+
+            if (purchase.IsSubmitted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+            }
+            if (purchase.IsStarted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+            }
+            if (purchase.IsCancelled)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsError)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsFinished)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+                if (!recent.Finish.HasValue) recent.Finish = now;
+            }
+            State.LastUpdate = now;
+            SaveState();
+        }
+
+        private async Task MakeNewPurchase()
+        {
+            var cid = await EnsureCid();
+            if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
+
+            Log($"Creating new purchase...");
+            var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
+            var purchaseId = response.PurchaseId;
+            var encodedCid = response.EncodedCid;
+            if (string.IsNullOrEmpty(purchaseId) ||
+                purchaseId == "Unable to encode manifest" ||
+                purchaseId == "Purchasing not available" ||
+                purchaseId == "Expiry required" ||
+                purchaseId == "Expiry needs to be in future" ||
+                purchaseId == "Expiry has to be before the request's end (now + duration)")
+            {
+                throw new InvalidOperationException(purchaseId);
+            }
+
+            var newPurchase = new WorkerPurchase
+            {
+                Created = DateTime.UtcNow,
+                Pid = purchaseId
+            };
+            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
+            State.EncodedCid = encodedCid.Id;
+            SaveState();
+            onNewPurchase();
+
+            Log($"New purchase created. PID: '{purchaseId}'.");
+            Log("Got Encoded-CID: " + encodedCid);
+            Log("Waiting for submit...");
+            Thread.Sleep(500);
+
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (DateTime.UtcNow < timeout)
+            {
+                Thread.Sleep(5000);
+                await UpdatePurchase(newPurchase);
+                if (newPurchase.Submitted.HasValue)
+                {
+                    Log("New purchase successfully submitted.");
+                    return;
+                }
+            }
+            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
+        private string GetFileTag(FileIndex filename)
+        {
+            return $"({filename.Index.ToString("00000")}) ";
+        }
+
+        [Serializable]
+        public class WorkerStatus
+        {
+            public DateTime LastUpdate { get; set; }
+            public string Cid { get; set; } = string.Empty;
+            public string EncodedCid { get; set; } = string.Empty;
+            public int FailureCounter { get; set; } = 0;
+            public string Error { get; set; } = string.Empty;
+            public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
+        }
+
+        [Serializable]
+        public class WorkerPurchase
+        {
+            public string Pid { get; set; } = string.Empty;
+            public DateTime Created { get; set; }
+            public DateTime? Submitted { get; set; }
+            public DateTime? Started { get; set; }
+            public DateTime? Expiry { get; set; }
+            public DateTime? Finish { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -0,0 +1,70 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkDispatcher
+    {
+        private readonly string[] files = Array.Empty<string>();
+        private readonly ILog log;
+        private int index = 0;
+        private int busyCount = 0;
+
+        public FolderWorkDispatcher(ILog log, string folder)
+        {
+            var fs = Directory.GetFiles(folder);
+            var result = new List<string>();
+            foreach (var f in fs)
+            {
+                if (!f.ToLowerInvariant().Contains(".json"))
+                {
+                    var info = new FileInfo(f);
+                    if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
+                    {
+                        result.Add(f);
+                    }
+                }
+            }
+            files = result.ToArray();
+            this.log = log;
+        }
+
+        public FileIndex GetFileToCheck()
+        {
+            if (busyCount > 0)
+            {
+                log.Log("");
+                log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
+                log.Log("");
+                ResetIndex();
+                Thread.Sleep(TimeSpan.FromMinutes(1));
+            }
+
+            var file = new FileIndex(files[index], index);
+            index = (index + 1) % files.Length;
+            return file;
+        }
+
+        public void ResetIndex()
+        {
+            index = 0;
+            busyCount = 0;
+        }
+
+        public void WorkerIsBusy()
+        {
+            busyCount++;
+        }
+    }
+
+    public class FileIndex
+    {
+        public FileIndex(string file, int index)
+        {
+            File = file;
+            Index = index;
+        }
+
+        public string File { get; }
+        public int Index { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -0,0 +1,136 @@
+using CodexOpenApi;
+using System.IO.Compression;
+using static AutoClient.Modes.FolderStore.FolderWorkOverview;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
+    {
+        private const string OverviewFilename = "codex_folder_saver_overview.json";
+        private readonly App app;
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
+            : base(app, folder, Path.Combine(folder, OverviewFilename))
+        {
+            this.app = app;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        protected override void OnNewState(WorkMonitorStatus newState)
+        {
+            newState.LastOverviewUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update(ICodexInstance instance)
+        {
+            var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
+
+            var total = 0;
+            var successful = 0;
+            var failed = 0;
+            foreach (var file in jsonFiles)
+            {
+                try
+                {
+                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
+                    total++;
+                    if (worker.IsCurrentlyRunning()) successful++;
+                    if (worker.IsCurrentlyFailed()) failed++;
+                }
+                catch (Exception exc)
+                {
+                    app.Log.Error("Exception in workoverview update: " + exc);
+                }
+            }
+
+            State.TotalFiles = total;
+            State.SuccessfulStored = successful;
+            State.StoreFailed = failed;
+            SaveState();
+
+            if (State.UncommitedChanges > 3)
+            {
+                State.UncommitedChanges = 0;
+                SaveState();
+
+                await CreateNewOverviewZip(jsonFiles, FilePath, instance);
+            }
+        }
+
+        public void MarkUncommitedChange()
+        {
+            State.UncommitedChanges++;
+            SaveState();
+        }
+
+        private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
+        {
+            Log("");
+            Log("");
+            Log("Creating new overview zipfile...");
+            var zipFilename = CreateZipFile(jsonFiles, filePath);
+
+            Log("Uploading to Codex...");
+            try
+            {
+                var codex = new CodexNode(app, instance);
+                var cid = await codex.UploadFile(zipFilename);
+                Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
+                Log("Requesting storage for it...");
+                var result = await codex.RequestStorage(cid);
+                Log("Storage requested. Purchase ID: " + result);
+
+                var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
+                File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
+                Log($">>> [{outFile}] has been updated. <<<");
+            }
+            catch (Exception exc)
+            {
+                Log("Failed to upload new overview zipfile: " + exc);
+            }
+            Log("");
+            Log("");
+        }
+
+        private string CreateZipFile(List<string> jsonFiles, string filePath)
+        {
+            var zipFilename = Guid.NewGuid().ToString() + ".zip";
+
+            using (var memoryStream = new MemoryStream())
+            {
+                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
+                {
+                    archive.CreateEntryFromFile(filePath, "overview.json");
+                    foreach (var file in jsonFiles)
+                    {
+                        archive.CreateEntryFromFile(file, Path.GetFileName(file));
+                    }
+                }
+
+                using (var fileStream = new FileStream(zipFilename, FileMode.Create))
+                {
+                    memoryStream.Seek(0, SeekOrigin.Begin);
+                    memoryStream.CopyTo(fileStream);
+                }
+            }
+            return zipFilename;
+        }
+
+        private void Log(string msg)
+        {
+            app.Log.Log(msg);
+        }
+
+        [Serializable]
+        public class WorkMonitorStatus
+        {
+            public int TotalFiles { get; set; }
+            public int SuccessfulStored { get; set; }
+            public int StoreFailed { get; set; }
+
+            public DateTime LastOverviewUpdate { get; set; }
+            public int UncommitedChanges { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/JsonBacked.cs

@@ -0,0 +1,58 @@
+using Newtonsoft.Json;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public abstract class JsonBacked<T> where T : new()
+    {
+        private readonly App app;
+
+        protected JsonBacked(App app, string folder, string filePath)
+        {
+            this.app = app;
+            Folder = folder;
+            FilePath = filePath;
+            LoadState();
+        }
+
+        private void LoadState()
+        {
+            try
+            {
+                if (!File.Exists(FilePath))
+                {
+                    State = new T();
+                    OnNewState(State);
+                    SaveState();
+                }
+                var text = File.ReadAllText(FilePath);
+                State = JsonConvert.DeserializeObject<T>(text)!;
+                if (State == null) throw new Exception("Didn't deserialize " + FilePath);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to load state: " + exc);
+            }
+        }
+
+        protected string Folder { get; }
+        protected string FilePath { get; }
+        protected T State { get; private set; } = default!;
+
+        protected virtual void OnNewState(T newState)
+        {
+        }
+
+        protected void SaveState()
+        {
+            try
+            {
+                var json = JsonConvert.SerializeObject(State);
+                File.WriteAllText(FilePath, json);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to save state: " + exc);
+            }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/PurchaseInfo.cs

@@ -0,0 +1,18 @@
+namespace AutoClient.Modes.FolderStore
+{
+    public class PurchaseInfo
+    {
+        public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
+        {
+            PurchaseDurationTotal = purchaseDurationTotal;
+            PurchaseDurationSafe = purchaseDurationSafe;
+
+            if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
+            if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
+            if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
+        }
+
+        public TimeSpan PurchaseDurationTotal { get; }
+        public TimeSpan PurchaseDurationSafe { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -0,0 +1,86 @@
+using AutoClient.Modes.FolderStore;
+
+namespace AutoClient.Modes
+{
+    public class FolderStoreMode : IMode
+    {
+        private readonly App app;
+        private readonly string folder;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly CancellationTokenSource cts = new CancellationTokenSource();
+        private Task checkTask = Task.CompletedTask;
+
+        public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
+        {
+            this.app = app;
+            this.folder = folder;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public void Start(ICodexInstance instance, int index)
+        {
+            checkTask = Task.Run(async () =>
+            {
+                try
+                {
+                    await RunChecker(instance);
+                }
+                catch (Exception ex)
+                {
+                    app.Log.Error("Exception in FolderStoreMode worker: " + ex);
+                    Environment.Exit(1);
+                }
+            });
+        }
+
+        private async Task RunChecker(ICodexInstance instance)
+        {
+            var i = 0;
+            while (!cts.IsCancellationRequested)
+            {
+                Thread.Sleep(2000);
+
+                var worker = await ProcessWorkItem(instance);
+                if (worker.FailureCounter > 5)
+                {
+                    throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
+                }
+                i++;
+
+                if (i > 5)
+                {
+                    i = 0;
+                    var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+                    await overview.Update(instance);
+                }
+            }
+        }
+
+        private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
+        {
+            var file = app.FolderWorkDispatcher.GetFileToCheck();
+            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
+            await worker.Update();
+            if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
+            return worker;
+        }
+
+        private void OnFileUploaded()
+        {
+        }
+
+        private void OnNewPurchase()
+        {
+            app.FolderWorkDispatcher.ResetIndex();
+
+            var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+            overview.MarkUncommitedChange();
+        }
+
+        public void Stop()
+        {
+            cts.Cancel();
+            checkTask.Wait();
+        }
+    }
+}

Tools/AutoClient/Modes/Mode.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AutoClient.Modes
+{
+    public interface IMode
+    {
+        void Start(ICodexInstance instance, int index);
+        void Stop();
+    }
+}

Tools/AutoClient/Modes/PurchasingMode.cs

@@ -1,32 +1,23 @@
-using CodexOpenApi;
-using Logging;
-using Utils;
+using Logging;
 
-namespace AutoClient
+namespace AutoClient.Modes
 {
-    public class CodexUser
+    public class PurchasingMode : IMode
     {
+        private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
         private readonly App app;
-        private readonly CodexApi codex;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly List<Purchaser> purchasers = new List<Purchaser>();
         private Task starterTask = Task.CompletedTask;
-        private readonly string nodeId = Guid.NewGuid().ToString();
 
-        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        public PurchasingMode(App app)
         {
             this.app = app;
-            this.codex = codex;
-            this.client = client;
-            this.address = address;
         }
 
-        public void Start(int index)
+        public void Start(ICodexInstance instance, int index)
         {
             for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
             {
-                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+                purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
             }
 
             var delayPerPurchaser =

Tools/AutoClient/Performance.cs

@@ -16,11 +16,13 @@ namespace AutoClient
             Log($"Download failed: {ex}");
         }
 
-        public void DownloadSuccessful(long size, TimeSpan time)
+        public void DownloadSuccessful(long? size, TimeSpan time)
         {
+            if (!size.HasValue) return;
+
             long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
             if (milliseconds < 1) milliseconds = 1;
-            long bytesPerSecond = 1000 * (size / milliseconds);
+            long bytesPerSecond = 1000 * (size.Value / milliseconds);
             Log($"Download successful: {bytesPerSecond} bytes per second");
         }
 

Tools/AutoClient/Program.cs

@@ -1,11 +1,14 @@
 using ArgsUniform;
 using AutoClient;
+using AutoClient.Modes;
+using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;
 
 public class Program
 {
     private readonly App app;
+    private readonly List<IMode> modes = new List<IMode>();
 
     public Program(Configuration config)
     {
@@ -31,36 +34,60 @@ public class Program
 
     public async Task Run()
     {
-        var codexUsers = await CreateUsers();
+        var codexInstances = await CreateCodexInstances();
 
         var i = 0;
-        foreach (var user in codexUsers)
+        foreach (var cdx in codexInstances)
         {
-            user.Start(i);
+            var mode = CreateMode();
+            modes.Add(mode);
+
+            mode.Start(cdx, i);
             i++;
         }
 
         app.Cts.Token.WaitHandle.WaitOne();
 
-        foreach (var user in codexUsers) user.Stop();
+        foreach (var mode in modes) mode.Stop();
+        modes.Clear();
 
         app.Log.Log("Done");
     }
 
-    private async Task<CodexUser[]> CreateUsers()
+    private IMode CreateMode()
+    {
+        if (!string.IsNullOrEmpty(app.Config.FolderToStore))
+        {
+            return CreateFolderStoreMode();
+        }
+
+        return new PurchasingMode(app);
+    }
+
+    private IMode CreateFolderStoreMode()
+    {
+        if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
+
+        return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
+            purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
+            purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
+        ));
+    }
+
+    private async Task<CodexInstance[]> CreateCodexInstances()
     {
         var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
-        var result = new List<CodexUser>();
+        var result = new List<CodexInstance>();
 
         foreach (var e in endpointStrs)
         {
-            result.Add(await CreateUser(e));
+            result.Add(await CreateCodexInstance(e));
         }
 
         return result.ToArray();
     }
 
-    private async Task<CodexUser> CreateUser(string endpoint)
+    private async Task<CodexInstance> CreateCodexInstance(string endpoint)
     {
         var splitIndex = endpoint.LastIndexOf(':');
         var host = endpoint.Substring(0, splitIndex);
@@ -72,6 +99,7 @@ public class Program
         );
 
         var client = new HttpClient();
+        client.Timeout = TimeSpan.FromMinutes(60.0);
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
@@ -79,7 +107,7 @@ public class Program
         await CheckCodex(codex);
         app.Log.Log("OK");
 
-        return new CodexUser(
+        return new CodexInstance(
             app,
             codex,
             client,
@@ -105,4 +133,4 @@ public class Program
     {
         Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
     }
-}
+}

Tools/AutoClient/Purchaser.cs

@@ -1,239 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public class Purchaser
-    {
-        private readonly App app;
-        private readonly string nodeId;
-        private readonly ILog log;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly CodexApi codex;
-        private Task workerTask = Task.CompletedTask;
-
-        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
-        {
-            this.app = app;
-            this.nodeId = nodeId;
-            this.log = log;
-            this.client = client;
-            this.address = address;
-            this.codex = codex;
-        }
-
-        public void Start()
-        {
-            workerTask = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            workerTask.Wait();
-        }
-
-        private async Task Worker()
-        {
-            log.Log("Worker started.");
-            while (!app.Cts.Token.IsCancellationRequested)
-            {
-                try
-                {
-                    var pid = await StartNewPurchase();
-                    await WaitTillFinished(pid);
-                    await DownloadForeignCid();
-                }
-                catch (Exception ex)
-                {
-                    log.Error("Worker failed with: " + ex);
-                    await Task.Delay(TimeSpan.FromHours(6));
-                }
-            }
-        }
-
-        private async Task DownloadForeignCid()
-        {
-            var cid = app.CidRepo.GetForeignCid(nodeId);
-            if (cid == null) return;
-            var size = app.CidRepo.GetSizeForCid(cid);
-            if (size == null) return;
-
-            try
-            {
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
-                {
-                    using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkAsync(cid);
-                    fileResponse.Stream.CopyTo(fileStream);
-                }
-                var time = sw.Elapsed;
-                File.Delete(filename);
-                app.Performance.DownloadSuccessful(size.Value, time);
-            }
-            catch (Exception ex)
-            {
-                app.Performance.DownloadFailed(ex);
-            }
-        }
-
-        private async Task<string> StartNewPurchase()
-        {
-            var file = await CreateFile();
-            var cid = await UploadFile(file);
-            return await RequestStorage(cid);
-        }
-
-        private async Task<string> CreateFile()
-        {
-            return await app.Generator.Generate();
-        }
-
-        private async Task<ContentId> UploadFile(string filename)
-        {
-            using var fileStream = File.OpenRead(filename);
-            try
-            {
-                var info = new FileInfo(filename);
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream);
-                var time = sw.Elapsed;
-                app.Performance.UploadSuccessful(info.Length, time);
-                app.CidRepo.Add(nodeId, cid.Id, info.Length);
-                return cid;
-            }
-            catch (Exception exc)
-            {
-                app.Performance.UploadFailed(exc);
-                throw;
-            }
-        }
-
-        private async Task<ContentId> UploadStream(FileStream fileStream)
-        {
-            log.Debug($"Uploading file...");
-            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
-
-            log.Debug($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        private async Task<string> RequestStorage(ContentId cid)
-        {
-            log.Debug("Requesting storage for " + cid.Id);
-            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
-            {
-                Collateral = app.Config.RequiredCollateral.ToString(),
-                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = app.Config.NumHosts,
-                Reward = app.Config.Price.ToString(),
-                ProofProbability = "15",
-                Tolerance = app.Config.HostTolerance
-            }, app.Cts.Token);
-
-            log.Debug("Purchase ID: " + result);
-
-            var encoded = await GetEncodedCid(result);
-            app.CidRepo.AddEncoded(cid.Id, encoded);
-
-            return result;
-        }
-
-        private async Task<string> GetEncodedCid(string pid)
-        {
-            try
-            {
-                var sp = (await GetStoragePurchase(pid))!;
-                return sp.Request.Content.Cid;
-            }
-            catch (Exception ex)
-            {
-                log.Error(ex.ToString());
-                throw;
-            }
-        }
-
-        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
-        {
-            // openapi still don't match code.
-            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-            if (string.IsNullOrEmpty(str)) return null;
-            return JsonConvert.DeserializeObject<StoragePurchase>(str);
-        }
-
-        private async Task WaitTillFinished(string pid)
-        {
-            try
-            {
-                var emptyResponseTolerance = 10;
-                while (!app.Cts.Token.IsCancellationRequested)
-                {
-                    var purchase = await GetStoragePurchase(pid);
-                    if (purchase == null)
-                    {
-                        await FixedShortDelay();
-                        emptyResponseTolerance--;
-                        if (emptyResponseTolerance == 0)
-                        {
-                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
-                            await ExpiryTimeDelay();
-                            return;
-                        }
-                        continue;
-                    }
-                    var status = purchase.State.ToLowerInvariant();
-                    if (status.Contains("cancel"))
-                    {
-                        app.Performance.StorageContractCancelled();
-                        return;
-                    }
-                    if (status.Contains("error"))
-                    {
-                        app.Performance.StorageContractErrored(purchase.Error);
-                        return;
-                    }
-                    if (status.Contains("finished"))
-                    {
-                        app.Performance.StorageContractFinished();
-                        return;
-                    }
-                    if (status.Contains("started"))
-                    {
-                        app.Performance.StorageContractStarted();
-                        await FixedDurationDelay();
-                    }
-
-                    await FixedShortDelay();
-                }
-            }
-            catch (Exception ex)
-            {
-                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
-                await ExpiryTimeDelay();
-            }
-        }
-
-        private async Task FixedDurationDelay()
-        {
-            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task ExpiryTimeDelay()
-        {
-            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task FixedShortDelay()
-        {
-            await Task.Delay(15 * 1000, app.Cts.Token);
-        }
-    }
-}

Tools/AutoClient/README.md

@@ -6,7 +6,7 @@ Can generate random images or random data of a specified size.
 
 ## How to run
 
-- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- dotnet 8.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
 - docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
 
 ## Configuration options

Tools/AutoClient/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/BiblioTech/AdminChecker.cs

@@ -1,5 +1,7 @@
-using Discord;
+using BiblioTech.Options;
+using Discord;
 using Discord.WebSocket;
+using Org.BouncyCastle.Utilities;
 
 namespace BiblioTech
 {
@@ -29,7 +31,19 @@ namespace BiblioTech
 
         public async Task SendInAdminChannel(string msg)
         {
-            await adminChannel.SendMessageAsync(msg);
+            await SendInAdminChannel(msg.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        public async Task SendInAdminChannel(string[] lines)
+        {
+            var chunker = new LineChunker(lines);
+            var chunks = chunker.GetChunks();
+            if (!chunks.Any()) return;
+
+            foreach (var chunk in chunks)
+            {
+                await adminChannel.SendMessageAsync(string.Join(Environment.NewLine, chunk));
+            }
         }
 
         public void SetAdminChannel(ISocketMessageChannel adminChannel)

Tools/BiblioTech/BaseCommand.cs

@@ -1,7 +1,6 @@
 using Discord.WebSocket;
 using BiblioTech.Options;
 using Discord;
-using k8s.KubeConfigModels;
 
 namespace BiblioTech
 {

Tools/BiblioTech/BiblioTech.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CodexCidChecker.cs

@@ -0,0 +1,202 @@
+using CodexOpenApi;
+using IdentityModel.Client;
+using Logging;
+using Utils;
+
+namespace BiblioTech
+{
+    public class CodexCidChecker
+    {
+        private static readonly string nl = Environment.NewLine;
+        private readonly Configuration config;
+        private readonly ILog log;
+        private CodexApi? currentCodexNode;
+
+        public CodexCidChecker(Configuration config, ILog log)
+        {
+            this.config = config;
+            this.log = log;
+        }
+
+        public async Task<CheckResponse> PerformCheck(string cid)
+        {
+            if (string.IsNullOrEmpty(config.CodexEndpoint))
+            {
+                return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
+            }
+
+            try
+            {
+                var codex = GetCodex();
+                var nodeCheck = await CheckCodex(codex);
+                if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
+
+                return await PerformCheck(codex, cid);
+            }
+            catch (Exception ex)
+            {
+                return new CheckResponse(false, "Internal server error", ex.ToString());
+            }
+        }
+
+        private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
+        {
+            try
+            {
+                var manifest = await codex.DownloadNetworkManifestAsync(cid);
+                return SuccessMessage(manifest);
+            }
+            catch (ApiException apiEx)
+            {
+                if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
+                if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
+                return UnexpectedReturnCode(apiEx.Response);
+            }
+            catch (Exception ex)
+            {
+                return UnexpectedException(ex);
+            }
+        }
+
+        #region Response formatting
+
+        private CheckResponse SuccessMessage(DataItem content)
+        {
+            return FormatResponse(
+                success: true,
+                title: $"Success: '{content.Cid}'",
+                error: "",
+                $"size: {content.Manifest.DatasetSize} bytes",
+                $"blockSize: {content.Manifest.BlockSize} bytes",
+                $"protected: {content.Manifest.Protected}"
+            );
+        }
+
+        private CheckResponse UnexpectedException(Exception ex)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Unexpected error",
+                error: ex.ToString(),
+                content: "Details will be sent to the bot-admin channel."
+            );
+        }
+
+        private CheckResponse UnexpectedReturnCode(string response)
+        {
+            var msg = "Unexpected return code. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Unexpected return code",
+                error: msg,
+                content: msg
+            );
+        }
+
+        private CheckResponse FailedToFetch(string response)
+        {
+            var msg = "Failed to download content. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Could not download content",
+                error: msg,
+                msg,
+                $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
+            );
+        }
+
+        private CheckResponse CidFormatInvalid(string response)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Invalid format",
+                error: "",
+                content: "Provided CID is not formatted correctly."
+            );
+        }
+
+        private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
+        {
+            var msg = string.Join(nl,
+                new string[]
+                {
+                    title,
+                    "```"
+                }
+                .Concat(content)
+                .Concat(new string[]
+                {
+                    "```"
+                })
+            ) + nl + nl;
+
+            return new CheckResponse(success, msg, error);
+        }
+
+        #endregion
+
+        #region Codex Node API 
+
+        private CodexApi GetCodex()
+        {
+            if (currentCodexNode == null) currentCodexNode = CreateCodex();
+            return currentCodexNode;
+        }
+
+        private async Task<bool> CheckCodex(CodexApi codex)
+        {
+            try
+            {
+                var info = await currentCodexNode!.GetDebugInfoAsync();
+                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
+                return true;
+            }
+            catch (Exception e)
+            {
+                log.Error(e.ToString());
+                return false;
+            }
+        }
+
+        private CodexApi CreateCodex()
+        {
+            var endpoint = config.CodexEndpoint;
+            var splitIndex = endpoint.LastIndexOf(':');
+            var host = endpoint.Substring(0, splitIndex);
+            var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
+
+            var address = new Address(
+                host: host,
+                port: port
+            );
+
+            var client = new HttpClient();
+            if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
+            {
+                var tokens = config.CodexEndpointAuth.Split(':');
+                if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
+                client.SetBasicAuthentication(tokens[0], tokens[1]);    
+            }
+
+            var codex = new CodexApi(client);
+            codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return codex;
+        }
+
+        #endregion
+    }
+
+    public class CheckResponse
+    {
+        public CheckResponse(bool success, string message, string error)
+        {
+            Success = success;
+            Message = message;
+            Error = error;
+        }
+
+        public bool Success { get; }
+        public string Message { get; }
+        public string Error { get; }
+    }
+}

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -0,0 +1,38 @@
+using BiblioTech.Options;
+
+namespace BiblioTech.Commands
+{
+    public class CheckCidCommand : BaseCommand
+    {
+        private readonly StringOption cidOption = new StringOption(
+            name: "cid",
+            description: "Codex Content-Identifier",
+            isRequired: true);
+        private readonly CodexCidChecker checker;
+
+        public CheckCidCommand(CodexCidChecker checker)
+        {
+            this.checker = checker;
+        }
+
+        public override string Name => "check";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Checks if content is available in the testnet.";
+        public override CommandOption[] Options => new[] { cidOption };
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            var user = context.Command.User;
+            var cid = await cidOption.Parse(context);
+            if (string.IsNullOrEmpty(cid))
+            {
+                await context.Followup("Option 'cid' was not received.");
+                return;
+            }
+
+            var response = await checker.PerformCheck(cid);
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
+            await context.Followup(response.Message);
+        }
+    }
+}

Tools/BiblioTech/Configuration.cs

@@ -38,6 +38,12 @@ namespace BiblioTech
         [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
         public int NoDiscord { get; set; } = 0;
 
+        [Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")]
+        public string CodexEndpoint { get; set; } = "http://localhost:8080";
+
+        [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
+        public string CodexEndpointAuth { get; set; } = "";
+
         public string EndpointsPath => Path.Combine(DataPath, "endpoints");
         public string UserDataPath => Path.Combine(DataPath, "users");
         public string LogPath => Path.Combine(DataPath, "logs");

Tools/BiblioTech/Program.cs

@@ -3,7 +3,6 @@ using BiblioTech.Commands;
 using BiblioTech.Rewards;
 using Discord;
 using Discord.WebSocket;
-using DiscordRewards;
 using Logging;
 
 namespace BiblioTech
@@ -81,6 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
+            var checker = new CodexCidChecker(Config, Log);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();
@@ -90,6 +90,7 @@ namespace BiblioTech
                 sprCommand,
                 associateCommand,
                 notifyCommand,
+                new CheckCidCommand(checker),
                 new AdminCommand(sprCommand, replacement)
             );
 

Tools/BiblioTech/Rewards/ChainEventsSender.cs

@@ -1,4 +1,5 @@
 using Discord.WebSocket;
+using DiscordRewards;
 using Logging;
 
 namespace BiblioTech.Rewards
@@ -16,24 +17,17 @@ namespace BiblioTech.Rewards
             this.eventsChannel = eventsChannel;
         }
 
-        public async Task ProcessChainEvents(string[] eventsOverview)
+        public async Task ProcessChainEvents(ChainEventMessage[] eventsOverview, string[] errors)
         {
+            await SendErrorsToAdminChannel(errors);
+
             if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
             try
             {
                 await Task.Run(async () =>
                 {
                     var users = Program.UserRepo.GetAllUserData();
-
-                    foreach (var e in eventsOverview)
-                    {
-                        if (!string.IsNullOrEmpty(e))
-                        {
-                            var @event = ApplyReplacements(users, e);
-                            await eventsChannel.SendMessageAsync(@event);
-                            await Task.Delay(1000);
-                        }
-                    }
+                    await SendChainEventsInOrder(eventsOverview, eventsChannel, users);
                 });
             }
             catch (Exception ex)
@@ -42,6 +36,37 @@ namespace BiblioTech.Rewards
             }
         }
 
+        private async Task SendErrorsToAdminChannel(string[] errors)
+        {
+            try
+            {
+                foreach (var error in errors)
+                {
+                    await Program.AdminChecker.SendInAdminChannel(error);
+                }
+            }
+            catch (Exception exc)
+            {
+                log.Error("Failed to send error messages to admin channel. " + exc);
+                Environment.Exit(1);
+            }
+        }
+
+        private async Task SendChainEventsInOrder(ChainEventMessage[] eventsOverview, SocketTextChannel eventsChannel, UserData[] users)
+        {
+            eventsOverview = eventsOverview.OrderBy(e => e.BlockNumber).ToArray();
+            foreach (var e in eventsOverview)
+            {
+                var msg = e.Message;
+                if (!string.IsNullOrEmpty(msg))
+                {
+                    var @event = ApplyReplacements(users, msg);
+                    await eventsChannel.SendMessageAsync(@event);
+                    await Task.Delay(300);
+                }
+            }
+        }
+
         private string ApplyReplacements(UserData[] users, string msg)
         {
             var result = ApplyUserAddressReplacements(users, msg);

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -31,7 +31,7 @@ namespace BiblioTech.Rewards
                 await ProcessRewards(rewards);
             }
 
-            await eventsSender.ProcessChainEvents(rewards.EventsOverview);
+            await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors);
         }
 
         private async Task ProcessRewards(GiveRewardsCommand rewards)

Tools/BiblioTech/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/CodexNetDeployer/Deployer.cs

@@ -161,7 +161,7 @@ namespace CodexNetDeployer
 
             Log("Starting metrics service...");
 
-            var runningContainer = ci.DeployMetricsCollector(startResults.Select(r => r.CodexNode).ToArray());
+            var runningContainer = ci.DeployMetricsCollector(scrapeInterval: TimeSpan.FromSeconds(10.0), startResults.Select(r => r.CodexNode).ToArray());
 
             Log("Metrics service started.");
 

Tools/CsvCombiner/CsvCombiner.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/CsvCombiner/Program.cs

@@ -0,0 +1,71 @@
+using Logging;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"];
+        var p = new Program(args[0]);
+        p.Run();
+    }
+
+    private static readonly ILog log = new ConsoleLog();
+    private string path;
+
+    private readonly Dictionary<string, List<string>> combine = new Dictionary<string, List<string>>();
+
+    public Program(string path)
+    {
+        this.path = path;
+    }
+
+    private void Run()
+    {
+        Log("Starting in " + path);
+
+        var files = Directory.GetFiles(path)
+            .Where(f => f.ToLowerInvariant().EndsWith(".csv")).ToArray();
+
+        foreach (var file in files)
+        {
+            AddToMap(file);
+        }
+
+        var i = 0;
+        foreach (var pair in combine)
+        {
+            var list = pair.Value;
+            list.Insert(0, pair.Key);
+
+            File.WriteAllLines(Path.Combine(path, "combine_" + i + ".csv"), list.ToArray());
+            i++;
+        }
+
+        Log("done");
+    }
+
+    private void AddToMap(string file)
+    {
+        var lines = File.ReadAllLines(file);
+        if (lines.Length > 1)
+        {
+            var header = lines[0];
+            var list = GetList(header);
+            list.AddRange(lines.Skip(1));
+        }
+    }
+
+    private List<string> GetList(string header)
+    {
+        if (!combine.ContainsKey(header))
+        {
+            combine.Add(header, new List<string>());
+        }
+        return combine[header];
+    }
+
+    private void Log(string msg)
+    {
+        log.Log(msg);
+    }
+}

Tools/KeyMaker/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/MarketInsights/AverageHistory.cs

@@ -1,6 +1,5 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
-using Nethereum.Model;
 using TestNetRewarder;
 using Utils;
 
@@ -40,7 +39,7 @@ namespace MarketInsights
 
         private MarketTimeSegment BuildContribution(TimeRange timeRange)
         {
-            var builder = new ContributionBuilder(timeRange);
+            var builder = new ContributionBuilder(appState.Log, timeRange);
             mux.Handlers.Add(builder);
             chainState.Update(timeRange.To);
             mux.Handlers.Remove(builder);

Tools/MarketInsights/ContributionBuilder.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin.ChainMonitor;
 using GethPlugin;
+using Logging;
 using System.Numerics;
 using Utils;
 
@@ -8,14 +9,16 @@ namespace MarketInsights
     public class ContributionBuilder : IChainStateChangeHandler
     {
         private readonly MarketTimeSegment segment = new MarketTimeSegment();
+        private readonly ILog log;
 
-        public ContributionBuilder(TimeRange timeRange)
+        public ContributionBuilder(ILog log, TimeRange timeRange)
         {
             segment = new MarketTimeSegment
             {
                 FromUtc = timeRange.From,
                 ToUtc = timeRange.To
             };
+            this.log = log;
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
@@ -51,6 +54,15 @@ namespace MarketInsights
         {
         }
 
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+            log.Error(msg);
+        }
+
         public MarketTimeSegment GetSegment()
         {
             return segment;

Tools/TestNetRewarder/BotClient.cs

@@ -18,7 +18,6 @@ namespace TestNetRewarder
         public async Task<bool> IsOnline()
         {
             var result = await HttpGet();
-            log.Log("Is DiscordBot online: " + result);
             return result == "Pong";
         }
 

Tools/TestNetRewarder/EmojiMaps.cs

@@ -0,0 +1,83 @@
+using Utils;
+
+namespace TestNetRewarder
+{
+    public class EmojiMaps
+    {
+        private readonly string[] emojis = new[]
+        {
+            // yellow
+            "😀",
+            "🌻",
+            "🍋",
+            "🧀",
+            "🌔",
+            "⭐",
+            "⚡",
+            "🏆",
+            // red
+            "💘",
+            "🦞",
+            "🌹",
+            "🍒",
+            "🫖", // teapot
+            "⛩",
+            "🚗",
+            "🔥",
+            // green
+            "🐊",
+            "🦎",
+            "🐛",
+            "🌳",
+            "🍀",
+            "🧩",
+            "🔋",
+            "♻",
+            // blue
+            "💙",
+            "🐳",
+            "🐟",
+            "🍉",
+            "🧊",
+            "🌐",
+            "⚓",
+            "🌀",
+        };
+
+        public string NewRequest => "🐟";
+        public string Started => "🦈";
+        public string SlotFilled => "🟢";
+        public string SlotFreed => "⭕";
+        public string SlotReservationsFull => "☑️";
+        public string Finished => "✅";
+        public string Cancelled => "🚫";
+        public string Failed => "❌";
+
+        public string StringToEmojis(string input, int outLength)
+        {
+            if (outLength < 1) outLength = 1;
+
+            var result = "";
+            var segmentLength = input.Length / outLength;
+            if (segmentLength < 1)
+            {
+                return StringToEmojis(input + input, outLength);
+            }
+            for (var i = 0; i < outLength; i++)
+            {
+                var segment = input.Substring(i * segmentLength, segmentLength);
+                result += SelectOne(segment);
+            }
+
+            return result;
+        }
+
+        private string SelectOne(string segment)
+        {
+            var index = 0;
+            foreach (var c in segment) index += Convert.ToInt32(c);
+            index = index % emojis.Length;
+            return emojis[index];
+        }
+    }
+}

Tools/TestNetRewarder/EventsFormatter.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
+using DiscordRewards;
 using GethPlugin;
 using System.Globalization;
 using System.Numerics;
@@ -10,24 +11,28 @@ namespace TestNetRewarder
     public class EventsFormatter : IChainStateChangeHandler
     {
         private static readonly string nl = Environment.NewLine;
-        private readonly List<string> events = new List<string>();
+        private readonly List<ChainEventMessage> events = new List<ChainEventMessage>();
+        private readonly List<string> errors = new List<string>();
+        private readonly EmojiMaps emojiMaps = new EmojiMaps();
 
-        public string[] GetEvents()
+        public ChainEventMessage[] GetEvents()
         {
             var result = events.ToArray();
             events.Clear();
             return result;
         }
 
-        public void AddError(string error)
+        public string[] GetErrors()
         {
-            AddBlock("📢 **Error**", error);
+            var result = errors.ToArray();
+            errors.Clear();
+            return result;
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
         {
             var request = requestEvent.Request;
-            AddRequestBlock(requestEvent, "New Request",
+            AddRequestBlock(requestEvent, $"{emojiMaps.NewRequest} New Request",
                 $"Client: {request.Client}",
                 $"Content: {request.Request.Content.Cid}",
                 $"Duration: {BigIntToDuration(request.Request.Ask.Duration)}",
@@ -42,27 +47,27 @@ namespace TestNetRewarder
 
         public void OnRequestCancelled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Cancelled");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Cancelled} Cancelled");
         }
 
         public void OnRequestFailed(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Failed");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Failed} Failed");
         }
 
         public void OnRequestFinished(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Finished");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Finished} Finished");
         }
 
         public void OnRequestFulfilled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Started");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Started} Started");
         }
 
         public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, "Slot Filled",
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFilled} Slot Filled",
                 $"Host: {host}",
                 $"Slot Index: {slotIndex}"
             );
@@ -70,24 +75,46 @@ namespace TestNetRewarder
 
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, "Slot Freed",
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFreed} Slot Freed",
                 $"Slot Index: {slotIndex}"
             );
         }
 
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotReservationsFull} Slot Reservations Full",
+                $"Slot Index: {slotIndex}"
+            );
+        }
+        
+        public void OnError(string msg)
+        {
+            errors.Add(msg);
+        }
+
         private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
         {
             var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
-            var title = $"{blockNumber} **{eventName}** `{requestEvent.Request.Request.Id}`";
-            AddBlock(title, content);
+            var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}";
+            AddBlock(requestEvent.Block.BlockNumber, title, content);
         }
 
-        private void AddBlock(string title, params string[] content)
+        private void AddBlock(ulong blockNumber, string title, params string[] content)
         {
-            events.Add(FormatBlock(title, content));
+            events.Add(FormatBlock(blockNumber, title, content));
         }
 
-        private string FormatBlock(string title, params string[] content)
+        private ChainEventMessage FormatBlock(ulong blockNumber, string title, params string[] content)
+        {
+            var msg = FormatBlockMessage(title, content);
+            return new ChainEventMessage
+            {
+                BlockNumber = blockNumber,
+                Message = msg
+            };
+        }
+
+        private string FormatBlockMessage(string title, string[] content)
         {
             if (content == null || !content.Any())
             {
@@ -113,6 +140,13 @@ namespace TestNetRewarder
             return utc.ToString("yyyy-MM-dd HH:mm:ss UTC", CultureInfo.InvariantCulture);
         }
 
+        private string FormatRequestId(RequestEvent requestEvent)
+        {
+            return
+                $"({emojiMaps.StringToEmojis(requestEvent.Request.Request.Id, 3)})" +
+                $"`{requestEvent.Request.Request.Id}`";
+        }
+
         private string BigIntToDuration(BigInteger big)
         {
             var span = TimeSpan.FromSeconds((int)big);

Tools/TestNetRewarder/Processor.cs

@@ -48,7 +48,7 @@ namespace TestNetRewarder
             {
                 var msg = "Exception processing time segment: " + ex;
                 log.Error(msg); 
-                eventsFormatter.AddError(msg);
+                eventsFormatter.OnError(msg);
                 throw;
             }
         }
@@ -58,8 +58,9 @@ namespace TestNetRewarder
             var numberOfChainEvents = chainState.Update(timeRange.To);
 
             var events = eventsFormatter.GetEvents();
+            var errors = eventsFormatter.GetErrors();
 
-            var request = builder.Build(events);
+            var request = builder.Build(events, errors);
             if (request.HasAny())
             {
                 await client.SendRewards(request);