wip tracking download fails

# Conflicts:
#	ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -40,6 +41,11 @@ namespace KubernetesWorkflow
 
         protected override void ProcessLine(string line)
         {
+            foreach (var replacement in BaseLog.replacements)
+            {
+                line = replacement.Apply(line);
+            }
+
             LogFile.WriteRaw(line);
         }
     }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -24,13 +24,12 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 

Framework/Logging/BaseLog.cs

@@ -16,7 +16,7 @@ namespace Logging
         public static bool EnableDebugLogging { get; set; } = false;
 
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
+        public static List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private LogFile? logFile;
 
         public BaseLog()

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -3,24 +3,41 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
         }
 
         public static T[] Shuffled<T>(T[] items)
         {
-            var result = new List<T>();
-            var source = items.ToList();
-            while (source.Any())
+            lock (@lock)
             {
-                result.Add(RandomUtils.PickOneRandom(source));
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
             }
-            return result.ToArray();
         }
     }
 }

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -48,24 +51,29 @@ namespace CodexContractsPlugin.ChainMonitor
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public void Update()
+        public int Update()
         {
-            Update(DateTime.UtcNow);
+            return Update(DateTime.UtcNow);
         }
 
-        public void Update(DateTime toUtc)
+        public int Update(DateTime toUtc)
         {
             var span = new TimeRange(TotalSpan.To, toUtc);
             var events = ChainEvents.FromTimeRange(contracts, span);
             Apply(events);
 
             TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
         }
 
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
             log.Log($"ChainState updating: {events.BlockInterval}");
 
@@ -108,7 +116,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -116,7 +124,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -124,7 +132,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -132,7 +140,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -141,13 +149,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -161,10 +177,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
 using NethereumWorkflow.BlockUtils;
 using Utils;
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "39-0C-32-A3-EA-90-4F-29-1C-67-12-F1-D5-BE-31-67-8D-90-43-1E-F2-02-63-5B-0C-49-F7-1E-E5-EC-F7-00";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -73,7 +73,7 @@ namespace CodexPlugin
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -82,25 +82,31 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            return mapper.Map(OnCodex(api => api.ListDataAsync("", "")));
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -189,7 +195,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,19 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
+        private const string DefaultDockerImage =
+        //"codexstorage/nim-codex:0.1.7-dist-tests"; // => 20/20: 17 seconds 10/10: 3 seconds
+        //"codexstorage/nim-codex:sha-2a25460-dist-tests"; // PR => 20/20: 17 seconds
+        //"thatbenbierens/nim-codex:blockexcpr1"; // PR with revert of "Fixes issue where only wants of type block are stored in peerContext" => 20/20: 17 seconds
+        //"thatbenbierens/nim-codex:blockexprecreate"; // v0.1.7 with patch => 20/20: 19 seconds
+        //"thatbenbierens/nim-codex:blockexprecreate016"; // v0.1.6 with patch => 20/20: 19 seconds 10/10: 2 seconds
+        //"thatbenbierens/nim-codex:blockexchprtinker7";
+        //"thatbenbierens/nim-codex:blkexc9"; // wow-fast
+
+        "thatbenbierens/nim-codex:asyncprofile5break"; // asynced trees.
+
+        //blocks are stored, blocks are resolved
+        //store-stream does not continue. node too busy???
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -244,6 +244,11 @@ namespace CodexPlugin
             Version = debugInfo.Version;
         }
 
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
             // The peer we want to connect is in a different pod.
@@ -259,10 +264,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,8 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
             };
         }
 
@@ -63,6 +63,26 @@ namespace CodexPlugin
             };
         }
 
+        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        {
+            return availabilities.Select(a => Map(a)).ToArray();
+        }
+
+        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        {
+            return new StorageAvailability
+            (
+                ToByteSize(availability.TotalSize),
+                ToTimespan(availability.Duration),
+                new TestToken(ToBigIng(availability.MinPrice)),
+                new TestToken(ToBigIng(availability.MaxCollateral))
+            )
+            {
+                Id = availability.Id,
+                FreeSpace = ToByteSize(availability.FreeSize),
+            };
+        }
+
         // TODO: Fix openapi spec for this call.
         //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
         //{
@@ -105,19 +125,6 @@ namespace CodexPlugin
         //    };
         //}
 
-        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
-        {
-            return new StorageAvailability(
-                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
-                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
-                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
-            )
-            {
-                Id = read.Id
-            };
-        }
-
         public CodexSpace Map(Space space)
         {
             return new CodexSpace
@@ -129,47 +136,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -222,5 +227,20 @@ namespace CodexPlugin
         {
             return t.TstWei.ToString("D");
         }
+
+        private BigInteger ToBigIng(string tokens)
+        {
+            return BigInteger.Parse(tokens);
+        }
+
+        private TimeSpan ToTimespan(string duration)
+        {
+            return TimeSpan.FromSeconds(Convert.ToInt32(duration));
+        }
+
+        private ByteSize ToByteSize(string size)
+        {
+            return new ByteSize(Convert.ToInt64(size));
+        }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -7,6 +7,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
+        StorageAvailability[] GetAvailabilities();
         IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
@@ -61,6 +62,14 @@ namespace CodexPlugin
             return response.Id;
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var result = codexAccess.GetAvailabilities();
+            Log($"Got {result.Length} availabilities:");
+            foreach (var a in result) a.Log(log);
+            return result;
+        }
+
         private void Log(string msg)
         {
             log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
@@ -81,6 +90,12 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         private void Unavailable()
         {
             FrameworkAssert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -84,10 +84,11 @@ namespace CodexPlugin
         public TimeSpan MaxDuration { get; }
         public TestToken MinPriceForTotalSpace { get; }
         public TestToken MaxCollateral { get; } 
+        public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
         {
-            log.Log($"Making storage available... (" +
+            log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
                 $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -23,6 +23,8 @@ components:
     Id:
       type: string
       description: 32bits identifier encoded in hex-decimal string.
+      minLength: 66
+      maxLength: 66
       example: 0x...
 
     BigInt:
@@ -81,33 +83,46 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
+
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
 
     DebugInfo:
       type: object
@@ -123,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -136,7 +155,7 @@ components:
           $ref: "#/components/schemas/Duration"
         minPrice:
           type: string
-          description: Minimum price to be paid (in amount of tokens) as decimal string
+          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
         maxCollateral:
           type: string
           description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@@ -168,7 +187,39 @@ components:
           $ref: "#/components/schemas/StorageRequest"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
+
+    SlotAgent:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        slotIndex:
+          type: string
+          description: Slot Index as decimal string
+        requestId:
+          $ref: "#/components/schemas/Id"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        reservation:
+          $ref: "#/components/schemas/Reservation"
+        state:
+          type: string
+          description: Description of the slot's
+          enum:
+            - SaleCancelled
+            - SaleDownloading
+            - SaleErrored
+            - SaleFailed
+            - SaleFilled
+            - SaleFilling
+            - SaleFinished
+            - SaleIgnored
+            - SaleInitialProving
+            - SalePayout
+            - SalePreparing
+            - SaleProving
+            - SaleUnknown
 
     Reservation:
       type: object
@@ -183,7 +234,7 @@ components:
           $ref: "#/components/schemas/Id"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
 
     StorageRequestCreation:
       type: object
@@ -259,6 +310,15 @@ components:
         state:
           type: string
           description: Description of the Request's state
+          enum:
+            - cancelled
+            - error
+            - failed
+            - finished
+            - pending
+            - started
+            - submitted
+            - unknown
         error:
           type: string
           description: If Request failed, then here is presented the error message
@@ -297,6 +357,19 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -308,15 +381,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -370,6 +443,21 @@ paths:
       summary: "Lists manifest CIDs stored locally in node."
       tags: [ Data ]
       operationId: listData
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       responses:
         "200":
           description: Retrieved list of content CIDs
@@ -382,6 +470,8 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
@@ -433,10 +523,36 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
+    post:
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetwork
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -459,6 +575,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -491,7 +633,7 @@ paths:
                   $ref: "#/components/schemas/Slot"
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/slots/{slotId}":
     get:
@@ -511,7 +653,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Slot"
+                $ref: "#/components/schemas/SlotAgent"
 
         "400":
           description: Invalid or missing SlotId
@@ -520,13 +662,13 @@ paths:
           description: Host is not in an active sale for the slot
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
       tags: [ Marketplace ]
-      operationId: getOfferedStorage
+      operationId: getAvailabilities
       responses:
         "200":
           description: Retrieved storage availabilities of the node
@@ -535,11 +677,11 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/SalesAvailability"
+                  $ref: "#/components/schemas/SalesAvailabilityREAD"
         "500":
           description: Error getting unused availabilities
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
     post:
       summary: "Offers storage for sale"
@@ -564,7 +706,7 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
   "/sales/availability/{id}":
     patch:
       summary: "Updates availability"
@@ -597,10 +739,10 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability/{id}/reservations":
-    patch:
+    get:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
@@ -628,7 +770,7 @@ paths:
         "500":
           description: Error getting reservations
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/storage/request/{cid}":
     post:
@@ -659,7 +801,7 @@ paths:
         "404":
           description: Request ID not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases":
     get:
@@ -676,7 +818,7 @@ paths:
                 items:
                   type: string
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases/{id}":
     get:
@@ -702,9 +844,9 @@ paths:
         "404":
           description: Purchase not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -722,7 +864,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -770,4 +912,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -2,9 +2,13 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/GethPlugin/EthTokenExtensions.cs

@@ -28,7 +28,13 @@
 
         public override string ToString()
         {
-            return $"{Eth} Eth";
+            var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
+
+            var tokens = new List<string>();
+            if (Eth > 0) tokens.Add($"{Eth} Eth");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
+
+            return string.Join(" + ", tokens);
         }
     }
 

ProjectPlugins/GethPlugin/GethNode.cs

@@ -57,7 +57,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/GethPlugin/GethPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");
@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";
@@ -80,7 +84,7 @@ namespace MetricsPlugin
     {
         public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -1,28 +1,46 @@
 using CodexPlugin;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace CodexTests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {
+        private readonly Dictionary<TestLifecycle, ICodexNode> bootstrapNodes = new Dictionary<TestLifecycle, ICodexNode>();
+
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
+            var tl = Get();
+            if (!bootstrapNodes.ContainsKey(tl))
+            {
+                bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace)));
+            }
         }
 
         [TearDown]
         public void TearDownBootstrapNode()
         {
-            BootstrapNode = null;
+            bootstrapNodes.Remove(Get());
         }
 
         protected override void OnCodexSetup(ICodexSetup setup)
         {
-            if (BootstrapNode != null) setup.WithBootstrapNode(BootstrapNode);
+            var node = BootstrapNode;
+            if (node != null) setup.WithBootstrapNode(node);
         }
 
-        protected ICodexNode? BootstrapNode { get; private set; }
-        
+        protected ICodexNode? BootstrapNode
+        {
+            get
+            {
+                var tl = Get();
+                if (bootstrapNodes.TryGetValue(tl, out var node))
+                {
+                    return node;
+                }
+                return null;
+            }
+        }
     }
 }

Tests/CodexTests/BasicTests/AsyncProfiling.cs

@@ -0,0 +1,85 @@
+using NUnit.Framework;
+using MetricsPlugin;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class AsyncProfiling : CodexDistTest
+    {
+        [Test]
+        public void AsyncProfileMetricsPlz()
+        {
+            var node = StartCodex(s => s.EnableMetrics());
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single();
+
+            var file = GenerateTestFile(100.MB());
+            node.UploadFile(file);
+
+            Thread.Sleep(10000);
+
+            var profilerMetrics = new AsyncProfileMetrics(metrics.GetAllMetrics());
+
+            var log = GetTestLog();
+            log.Log($"{nameof(profilerMetrics.CallCount)} = {profilerMetrics.CallCount.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTime)} = {profilerMetrics.ExecTime.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTimeWithChildren)} = {profilerMetrics.ExecTimeWithChildren.Highest()}");
+            log.Log($"{nameof(profilerMetrics.SingleExecTimeMax)} = {profilerMetrics.SingleExecTimeMax.Highest()}");
+            log.Log($"{nameof(profilerMetrics.WallTime)} = {profilerMetrics.WallTime.Highest()}");
+        }
+    }
+
+    public class AsyncProfileMetrics
+    {
+        public AsyncProfileMetrics(Metrics metrics)
+        {
+            CallCount = CreateMetric(metrics, "chronos_call_count_total");
+            ExecTime = CreateMetric(metrics, "chronos_exec_time_total");
+            ExecTimeWithChildren = CreateMetric(metrics, "chronos_exec_time_with_children_total");
+            SingleExecTimeMax = CreateMetric(metrics, "chronos_single_exec_time_max");
+            WallTime = CreateMetric(metrics, "chronos_wall_time_total");
+        }
+
+        public AsyncProfileMetric CallCount { get; }
+        public AsyncProfileMetric ExecTime { get; }
+        public AsyncProfileMetric ExecTimeWithChildren { get; }
+        public AsyncProfileMetric SingleExecTimeMax { get; }
+        public AsyncProfileMetric WallTime { get; }
+
+        private static AsyncProfileMetric CreateMetric(Metrics metrics, string name)
+        {
+            var sets = metrics.Sets.Where(s => s.Name == name).ToArray();
+            return new AsyncProfileMetric(sets);
+        }
+    }
+
+    public class AsyncProfileMetric
+    {
+        private readonly MetricsSet[] metricsSets;
+
+        public AsyncProfileMetric(MetricsSet[] metricsSets)
+        {
+            this.metricsSets = metricsSets;
+        }
+
+        public MetricsSet Highest()
+        {
+            MetricsSet? result = null;
+            var highest = double.MinValue;
+            foreach (var metric in metricsSets)
+            {
+                foreach (var value in metric.Values)
+                {
+                    if (value.Value > highest)
+                    {
+                        highest = value.Value;
+                        result = metric;
+                    }
+                }
+            }
+
+            if (result == null) throw new Exception("None were highest");
+            return result;
+        }
+    }
+}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -36,7 +36,7 @@ namespace CodexTests.BasicTests
             var primary2 = group2[0];
             var secondary2 = group2[1];
 
-            var metrics = Ci.GetMetricsFor(primary, primary2);
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(10), primary, primary2);
 
             primary.ConnectToPeer(secondary);
             primary2.ConnectToPeer(secondary2);

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -97,6 +97,12 @@ namespace CodexTests.BasicTests
 
             purchaseContract.WaitForStorageContractStarted();
 
+            var availabilities = hosts.Select(h => h.Marketplace.GetAvailabilities()).ToArray();
+            if (availabilities.All(h => h.All(a => a.FreeSpace.SizeInBytes == a.TotalSpace.SizeInBytes)))
+            {
+                Assert.Fail("Host availabilities were not used.");
+            }
+
             var request = GetOnChainStorageRequest(contracts, geth);
             AssertStorageRequest(request, purchase, contracts, client);
             AssertContractSlot(contracts, request, 0);
@@ -107,47 +113,6 @@ namespace CodexTests.BasicTests
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
         }
 
-        [Test]
-        [Ignore("Integrated into MarketplaceExample to speed up testing.")]
-        public void CanDownloadContentFromContractCid()
-        {
-            var fileSize = 10.MB();
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
-            var testFile = CreateFile(fileSize);
-
-            var client = StartCodex(s => s
-                .WithName("Client")
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), 10.Tst())));
-
-            var uploadCid = client.UploadFile(testFile);
-
-            var purchase = new StoragePurchaseRequest(uploadCid)
-            {
-                PricePerSlotPerSecond = 2.TstWei(),
-                RequiredCollateral = 10.TstWei(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
-            };
-
-            var purchaseContract = client.Marketplace.RequestStorage(purchase);
-            var contractCid = purchaseContract.ContentId;
-            Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
-
-            // Download both from client.
-            testFile.AssertIsEqual(client.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(client.DownloadContent(contractCid));
-
-            // Download both from another node.
-            var downloader = StartCodex(s => s.WithName("Downloader"));
-            testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
-        }
-
         private TrackedFile CreateFile(ByteSize fileSize)
         {
             var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,5 +1,7 @@
 using CodexPlugin;
+using FileUtils;
 using NUnit.Framework;
+using System.Diagnostics;
 using Utils;
 
 namespace CodexTests.BasicTests
@@ -10,11 +12,44 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
+        }
+
+        [Test]
+        public void InterruptUploadTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
         }
 
         private void PerformOneClientTest(ICodexNode primary)

Tests/CodexTests/BasicTests/PyramidTests.cs

@@ -17,7 +17,7 @@ namespace CodexTests.BasicTests
         public void PyramidTest()
         {
             var size = 5.MB();
-            var numberOfLayers = 4;
+            var numberOfLayers = 3;
 
             var bottomLayer = StartLayers(numberOfLayers);
 

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -22,24 +22,6 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloadedFile);
         }
 
-        [Test]
-        [CreateTranscript(nameof(SwarmTest))]
-        public void SwarmTest()
-        {
-            var uploader = StartCodex(s => s.WithName("uploader"));
-            var downloaders = StartCodex(5, s => s.WithName("downloader"));
-
-            var file = GenerateTestFile(100.MB());
-            var cid = uploader.UploadFile(file);
-
-            var result = Parallel.ForEach(downloaders, d =>
-            {
-                d.DownloadContent(cid);
-            });
-
-            Assert.That(result.IsCompleted);
-        }
-
         [Test]
         public void DownloadingUnknownCidDoesNotCauseCrash()
         {

Tests/CodexTests/CodexTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs

@@ -0,0 +1,228 @@
+using CodexPlugin;
+using FileUtils;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class MultiswarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        public void Multiswarm(
+            [Values(3, 5)] int numFiles,
+            [Values(5, 20)] int fileSizeMb,
+            [Values(1)] int uploadersPerFile,
+            [Values(3)] int downloadersPerFile,
+            [Values(1)] int maxUploadsPerNode,
+            [Values(2, 3)] int maxDownloadsPerNode
+        )
+        {
+            var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            Assert.That(plan.NodePlans.Count, Is.LessThan(30));
+
+            RunThePlan(plan, fileSizeMb);
+        }
+
+        private void RunThePlan(Plan plan, int fileSizeMb)
+        {
+            foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB());
+            var nodes = StartCodex(plan.NodePlans.Count);
+            for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i];
+
+            // Upload all files to their nodes.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var uploader in filePlan.Uploaders)
+                {
+                    filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!);
+                }
+            }
+
+            Thread.Sleep(5000); // Everything is processed and announced.
+
+            // Start all downloads (almost) simultaneously.
+            var tasks = new List<Task>();
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloader in filePlan.Downloaders)
+                {
+                    tasks.Add(Task.Run(() =>
+                    {
+                        var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!);
+                        lock (filePlan.DownloadedFiles)
+                        {
+                            filePlan.DownloadedFiles.Add(downloadedFile);
+                        }
+                    }));
+                }
+            }
+
+            Task.WaitAll(tasks.ToArray());
+
+            // Assert all files are correct.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloadedFile in filePlan.DownloadedFiles)
+                {
+                    filePlan.File!.AssertIsEqual(downloadedFile);
+                }
+            }
+        }
+
+        private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            plan.Initialize();
+            plan.LogPlan(GetTestLog());
+            return plan;
+        }
+    }
+
+    public class FilePlan
+    {
+        public FilePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public TrackedFile? File { get; set; }
+        public ContentId? Cid { get; set; }
+        public List<TrackedFile?> DownloadedFiles { get; } = new List<TrackedFile?>();
+        public List<NodePlan> Uploaders { get; } = new List<NodePlan>();
+        public List<NodePlan> Downloaders { get; } = new List<NodePlan>();
+
+        public override string ToString()
+        {
+            return $"FilePlan[{Number}] " +
+                $"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " +
+                $"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class NodePlan
+    {
+        public NodePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public ICodexNode? Node { get; set; }
+        public List<FilePlan> Uploads { get; } = new List<FilePlan>();
+        public List<FilePlan> Downloads { get; } = new List<FilePlan>();
+
+        public bool Contains(FilePlan plan)
+        {
+            return Uploads.Contains(plan) || Downloads.Contains(plan);
+        }
+
+        public override string ToString()
+        {
+            return $"NodePlan[{Number}] " +
+                $"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " +
+                $"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class Plan
+    {
+        private readonly int numFiles;
+        private readonly int uploadersPerFile;
+        private readonly int downloadersPerFile;
+        private readonly int maxUploadsPerNode;
+        private readonly int maxDownloadsPerNode;
+
+        public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            this.numFiles = numFiles;
+            this.uploadersPerFile = uploadersPerFile;
+            this.downloadersPerFile = downloadersPerFile;
+            this.maxUploadsPerNode = maxUploadsPerNode;
+            this.maxDownloadsPerNode = maxDownloadsPerNode;
+        }
+
+        public List<FilePlan> FilePlans { get; } = new List<FilePlan>();
+        public List<NodePlan> NodePlans { get; } = new List<NodePlan>();
+
+        public void Initialize()
+        {
+            for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i));
+            foreach (var filePlan in FilePlans)
+            {
+                while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan);
+                while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan);
+            }
+
+            CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number));
+            CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number));
+
+            foreach (var filePlan in FilePlans)
+            {
+                Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile));
+                Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile));
+            }
+            foreach (var nodePlan in NodePlans)
+            {
+                Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode));
+                Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode));
+            }
+        }
+
+        public void LogPlan(ILog log)
+        {
+            log.Log("The plan:");
+            log.Log("Input:");
+            log.Log($"numFiles: {numFiles}");
+            log.Log($"uploadersPerFile: {uploadersPerFile}");
+            log.Log($"downloadersPerFile: {downloadersPerFile}");
+            log.Log($"maxUploadsPerNode: {maxUploadsPerNode}");
+            log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}");
+            log.Log("Setup:");
+            log.Log($"number of nodes: {NodePlans.Count}");
+            foreach (var filePlan in FilePlans) log.Log(filePlan.ToString());
+            foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString());
+        }
+
+        private void AddDownloader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateDownloaderNode(filePlan);
+            filePlan.Downloaders.Add(nodePlan);
+            nodePlan.Downloads.Add(filePlan);
+        }
+
+        private void AddUploader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateUploaderNode(filePlan);
+            filePlan.Uploaders.Add(nodePlan);
+            nodePlan.Uploads.Add(filePlan);
+        }
+
+        private NodePlan GetOrCreateDownloaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+
+        private NodePlan GetOrCreateUploaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+    }
+}

Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs

@@ -0,0 +1,25 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class SwarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [CreateTranscript("swarm_retransmit")]
+        public void DetectBlockRetransmits(
+            [Values(1, 5, 10, 20)] int fileSize,
+            [Values(3, 5, 10, 20)] int numNodes
+        )
+        {
+            var nodes = StartCodex(numNodes);
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = nodes[0].UploadFile(file);
+
+            var tasks = nodes.Select(n => Task.Run(() => n.DownloadContent(cid))).ToArray();
+            Task.WaitAll(tasks);
+        }
+    }
+}

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests
         private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
         private readonly List<ulong> rewardsSeen = new List<ulong>();
         private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
-        private readonly List<string> receivedEvents = new List<string>();
+        private readonly List<ChainEventMessage> receivedEvents = new List<ChainEventMessage>();
  
         [Test]
         [DontDownloadLogs]
@@ -73,13 +73,18 @@ namespace CodexTests.UtilityTests
 
         private void AssertEventOccurance(string msg, int expectedCount)
         {
-            Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
+            Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount),
                 $"Event '{msg}' did not occure correct number of times.");
         }
 
         private void OnCommand(string timestamp, GiveRewardsCommand call)
         {
             Log($"<API call {timestamp}>");
+            foreach (var e in call.EventsOverview)
+            {
+                Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order.");
+            }
+
             receivedEvents.AddRange(call.EventsOverview);
             foreach (var e in call.EventsOverview)
             {

Tests/DistTestCore/DistTestCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Tests/DistTestCore/TestLifecycle.cs

@@ -22,6 +22,7 @@ namespace DistTestCore
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
+            TestNamespace = testNamespace;
             TestStart = DateTime.UtcNow;
 
             entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
@@ -36,6 +37,7 @@ namespace DistTestCore
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public string TestNamespace { get; }
         public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
 

Tests/FrameworkTests/FrameworkTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -18,6 +18,7 @@
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
   </ItemGroup>
 
 </Project>

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -19,11 +19,20 @@ namespace FrameworkTests.NethereumWorkflow
         {
             var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30);
             blocks = new Dictionary<ulong, Block>();
-            
+
+            Block? prev = null;
             for (ulong i = 0; i < 30; i++)
             {
                 ulong d = 100 + i;
-                blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2)));
+                var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2));
+                blocks.Add(d, newBlock);
+
+                if (prev != null)
+                {
+                    prev.Next = newBlock;
+                    newBlock.Previous = prev;
+                }
+                prev = newBlock;
             }
         }
 
@@ -99,23 +108,23 @@ namespace FrameworkTests.NethereumWorkflow
         }
 
         [Test]
-        public void FailsToFindBlockBeforeFrontOfChain()
+        public void FindsGenesisBlockAtFrontOfChain()
         {
             var first = blocks.First().Value;
 
-            var notFound = finder.GetHighestBlockNumberBefore(first.Time);
+            var firstNumber = finder.GetHighestBlockNumberBefore(first.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(firstNumber, Is.EqualTo(first.Number));
         }
 
         [Test]
-        public void FailsToFindBlockAfterTailOfChain()
+        public void FindsCurrentBlockAtTailOfChain()
         {
             var last = blocks.Last().Value;
 
-            var notFound = finder.GetLowestBlockNumberAfter(last.Time);
+            var lastNumber = finder.GetLowestBlockNumberAfter(last.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(lastNumber, Is.EqualTo(last.Number));
         }
 
         [Test]
@@ -143,13 +152,27 @@ namespace FrameworkTests.NethereumWorkflow
         {
             foreach (var pair in blocks)
             {
-                finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
-                finder.GetHighestBlockNumberBefore(pair.Value.Time);
-                finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
+                var block = pair.Value;
 
-                finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
-                finder.GetLowestBlockNumberAfter(pair.Value.Time);
-                finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
+                AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter));
+
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore));
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time));
+                AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter));
+            }
+        }
+
+        private void AssertLink(Block? expected, ulong? actual)
+        {
+            if (expected == null)
+            {
+                Assert.That(actual, Is.Null);
+            }
+            else
+            {
+                Assert.That(expected.Number, Is.EqualTo(actual!.Value));
             }
         }
     }
@@ -167,6 +190,9 @@ namespace FrameworkTests.NethereumWorkflow
         public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
         public DateTime JustAfter { get { return Time.AddSeconds(1); } }
 
+        public Block? Next { get; set; }
+        public Block? Previous { get; set; }
+
         public override string ToString()
         {
             return $"[{Number}]";

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -1,135 +0,0 @@
-using Logging;
-using NUnit.Framework;
-using OverwatchTranscript;
-
-namespace FrameworkTests.OverwatchTranscript
-{
-    [TestFixture]
-    public class TranscriptTests
-    {
-        private const string TranscriptFilename = "testtranscript.owts";
-        private const string HeaderKey = "testHeader";
-        private const string HeaderData = "abcdef";
-        private const string EventData0 = "12345";
-        private const string EventData1 = "678";
-        private const string EventData2 = "90";
-        private const string EventData3 = "-=";
-        private readonly DateTime t0 = DateTime.UtcNow;
-        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
-        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
-
-        [Test]
-        public void WriteAndRun()
-        {
-            WriteTranscript();
-            ReadTranscript();
-
-            File.Delete(TranscriptFilename);
-        }
-
-        private void WriteTranscript()
-        {
-            var log = new ConsoleLog();
-            var writer = Transcript.NewWriter(log);
-
-            writer.AddHeader(HeaderKey, new TestHeader
-            {
-                HeaderData = HeaderData
-            });
-
-            writer.Add(t0, new MyEvent
-            {
-                EventData = EventData0
-            });
-            writer.Add(t2, new MyEvent
-            {
-                EventData = EventData3
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData1
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData2
-            });
-
-            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
-
-            writer.Write(TranscriptFilename);
-        }
-
-        private void ReadTranscript()
-        {
-            var reader = Transcript.NewReader(TranscriptFilename);
-
-            var header = reader.GetHeader<TestHeader>(HeaderKey);
-            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
-            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
-            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
-            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
-            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
-
-            var moments = new List<ActivateMoment>();
-            var events = new List<ActivateEvent<MyEvent>>();
-            reader.AddMomentHandler(moments.Add);
-            reader.AddEventHandler<MyEvent>(events.Add);
-
-
-            Assert.That(moments.Count, Is.EqualTo(0));
-            Assert.That(events.Count, Is.EqualTo(0));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(1));
-            Assert.That(events.Count, Is.EqualTo(1));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(2));
-            Assert.That(events.Count, Is.EqualTo(3));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
-            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
-            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
-
-            AssertEvent(events[0], utc: t0, duration: t1 - t0, index: 0, data: EventData0);
-            AssertEvent(events[1], utc: t1, duration: t2 - t1, index: 1, data: EventData1);
-            AssertEvent(events[2], utc: t1, duration: t2 - t1, index: 1, data: EventData2);
-            AssertEvent(events[3], utc: t2, duration: null, index: 2, data: EventData3);
-
-            reader.Close();
-        }
-
-        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
-        {
-            Assert.That(m.Utc, Is.EqualTo(utc));
-            Assert.That(m.Duration, Is.EqualTo(duration));
-            Assert.That(m.Index, Is.EqualTo(index));
-        }
-
-        private void AssertEvent(ActivateEvent<MyEvent> e, DateTime utc, TimeSpan? duration, int index, string data)
-        {
-            Assert.That(e.Moment.Utc, Is.EqualTo(utc));
-            Assert.That(e.Moment.Duration, Is.EqualTo(duration));
-            Assert.That(e.Moment.Index, Is.EqualTo(index));
-            Assert.That(e.Payload.EventData, Is.EqualTo(data));
-        }
-    }
-
-    public class TestHeader
-    {
-        public string HeaderData { get; set; } = string.Empty;
-    }
-
-    public class MyEvent
-    {
-        public string EventData { get; set; } = string.Empty;
-    }
-}

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptLargeTest.cs

@@ -2,7 +2,7 @@
 using NUnit.Framework;
 using OverwatchTranscript;
 
-namespace FrameworkTests.OverwatchTranscript
+namespace FrameworkTests.OverwatchTranscriptTests
 {
     [TestFixture]
     public class TranscriptLargeTests

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptTests.cs

@@ -0,0 +1,229 @@
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using OverwatchTranscript;
+using System.IO.Compression;
+
+namespace FrameworkTests.OverwatchTranscriptTests
+{
+    [TestFixture]
+    public class TranscriptTests
+    {
+        private const string TranscriptFilename = "testtranscript.owts";
+        private const string HeaderKey = "testHeader";
+        private const string HeaderData = "abcdef";
+        private const string EventData0 = "12345";
+        private const string EventData1 = "678";
+        private const string EventData2 = "90";
+        private const string EventData3 = "-=";
+        private readonly DateTime t0 = DateTime.UtcNow;
+        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
+        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
+
+        [Test]
+        [Combinatorial]
+        public void WriteAndRun()
+        {
+            WriteTranscript();
+            AssertFileContent();
+            ReadTranscript();
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private void WriteTranscript()
+        {
+            var log = new ConsoleLog();
+            var writer = Transcript.NewWriter(log);
+
+            writer.AddHeader(HeaderKey, new TestHeader
+            {
+                HeaderData = HeaderData
+            });
+
+            writer.Add(t0, new MyEvent
+            {
+                EventData = EventData0
+            });
+            writer.Add(t2, new MyEvent
+            {
+                EventData = EventData3
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData1
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData2
+            });
+
+            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
+
+            writer.Write(TranscriptFilename);
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var header = reader.GetHeader<TestHeader>(HeaderKey);
+            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
+            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
+
+            var moments = new List<ActivateMoment>();
+            var events = new List<ActivateEvent<MyEvent>>();
+            reader.AddMomentHandler(moments.Add);
+            reader.AddEventHandler<MyEvent>(events.Add);
+
+            var timeout = 10;
+            while (moments.Count < 3 && events.Count < 4)
+            {
+                Thread.Sleep(10);
+                reader.Next();
+
+                timeout--;
+                if (timeout == 0) Assert.Fail("Events not received.");
+            }
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
+            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
+            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
+
+            AssertEvent(events, utc: t0, duration: t1 - t0, index: 0, data: EventData0);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData2);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData1);
+            AssertEvent(events, utc: t2, duration: null, index: 2, data: EventData3);
+
+            reader.Close();
+        }
+
+        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
+        {
+            Assert.That(m.Utc, Is.EqualTo(utc));
+            Assert.That(m.Duration, Is.EqualTo(duration));
+            Assert.That(m.Index, Is.EqualTo(index));
+        }
+
+        private void AssertEvent(List<ActivateEvent<MyEvent>> events, DateTime utc, TimeSpan? duration, int index, string data)
+        {
+            var e = events.SingleOrDefault(e => e.Moment.Utc == utc && e.Payload.EventData == data);
+            if (e == null) Assert.Fail("Event not found");
+
+            Assert.That(e!.Moment.Utc, Is.EqualTo(utc));
+            Assert.That(e!.Moment.Duration, Is.EqualTo(duration));
+            Assert.That(e!.Moment.Index, Is.EqualTo(index));
+            Assert.That(e!.Payload.EventData, Is.EqualTo(data));
+        }
+
+        private void AssertFileContent()
+        {
+            using var zip = ZipFile.OpenRead(TranscriptFilename);
+            Assert.That(zip.Entries.Count, Is.EqualTo(2));
+            foreach (var entry in zip.Entries)
+            {
+                if (entry.Name == "transcript.json")
+                {
+                    var transcript = ZipEntryJson<OverwatchTranscript.OverwatchTranscript>(entry);
+                    AssertTranscript(transcript);
+                }
+                else
+                {
+                    var moments = ZipEntryToMoments(entry);
+                    AssertMoments(moments);
+                }
+            }
+        }
+
+        private void AssertTranscript(OverwatchTranscript.OverwatchTranscript transcript)
+        {
+            Assert.That(transcript.Header.Common.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.Header.Common.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.Header.Common.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.Header.Common.LatestUtc, Is.EqualTo(t2));
+            Assert.That(transcript.Header.Entries.Length, Is.EqualTo(1));
+            Assert.That(transcript.Header.Entries[0].Key, Is.EqualTo(HeaderKey));
+            Assert.That(transcript.Header.Entries[0].Value, Is.EqualTo("{\"HeaderData\":\"abcdef\"}"));
+
+            Assert.That(transcript.MomentReferences.Length, Is.EqualTo(1));
+            Assert.That(transcript.MomentReferences[0].NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.MomentReferences[0].NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.MomentReferences[0].EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.MomentReferences[0].LatestUtc, Is.EqualTo(t2));
+        }
+
+        private void AssertMoments(OverwatchMoment[] moments)
+        {
+            Assert.That(moments.Length, Is.EqualTo(3));
+
+            Assert.That(moments[0].Utc, Is.EqualTo(t0));
+            Assert.That(moments[0].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[0].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[0].Events[0].Payload, Is.EqualTo("{\"EventData\":\"12345\"}"));
+
+            Assert.That(moments[1].Utc, Is.EqualTo(t1));
+            Assert.That(moments[1].Events.Length, Is.EqualTo(2));
+            Assert.That(moments[1].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[1].Events[1].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+
+            // output order is not guaranteed:
+            var payloads = moments[1].Events.Select(e => e.Payload).ToArray();
+            CollectionAssert.AreEquivalent(new[]
+            {
+                "{\"EventData\":\"90\"}",
+                "{\"EventData\":\"678\"}"
+            }, payloads);
+
+            Assert.That(moments[2].Utc, Is.EqualTo(t2));
+            Assert.That(moments[2].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[2].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[2].Events[0].Payload, Is.EqualTo("{\"EventData\":\"-=\"}"));
+        }
+
+        private T ZipEntryJson<T>(ZipArchiveEntry? entry)
+        {
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+            var json = reader.ReadToEnd();
+            var result = JsonConvert.DeserializeObject<T>(json);
+            if (result == null) Assert.Fail("didn't deserialize");
+            return result!;
+        }
+
+        private OverwatchMoment[] ZipEntryToMoments(ZipArchiveEntry? entry)
+        {
+            var result = new List<OverwatchMoment>();
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+
+            var line = reader.ReadLine();
+            while (!string.IsNullOrEmpty(line))
+            {
+                var moment = JsonConvert.DeserializeObject<OverwatchMoment>(line);
+                if (moment != null) result.Add(moment);
+                line = reader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+    }
+
+    public class TestHeader
+    {
+        public string HeaderData { get; set; } = string.Empty;
+    }
+
+    public class MyEvent
+    {
+        public string EventData { get; set; } = string.Empty;
+    }
+}

Tests/FrameworkTests/Utils/EmojiMapsTests.cs

@@ -0,0 +1,28 @@
+using NUnit.Framework;
+using System.Text;
+using TestNetRewarder;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EmojiMapsTests
+    {
+        private readonly Random random = new Random();
+        private readonly EmojiMaps maps = new EmojiMaps();
+
+        [Test]
+        public void GeneratesConsistentStrings(
+            [Values(1, 5, 10, 20)] int inputLength,
+            [Values(1, 2, 3, 5)] int outLength)
+        {
+            var buffer = new byte[inputLength];
+            random.NextBytes(buffer);
+            var input = Encoding.ASCII.GetString(buffer);
+
+            var out1 = maps.StringToEmojis(input, outLength);
+            var out2 = maps.StringToEmojis(input, outLength);
+            
+            Assert.That(out1, Is.EqualTo(out2));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncoding.cs

@@ -0,0 +1,334 @@
+namespace FrameworkTests.Utils
+{
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public RunUpdate ExpandToInclude(int index)
+        {
+            if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}");
+            if (index == (Start + Length))
+            {
+                Length++;
+                return new RunUpdate();
+            }
+            if (index == (Start - 1))
+            {
+                return new RunUpdate(
+                    newRuns: [new Run(Start - 1, Length + 1)],
+                    removeRuns: [this]
+                );
+            }
+            throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}");
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(
+                        newRuns: Array.Empty<Run>(),
+                        removeRuns: [this]
+                    );
+                }
+                return new RunUpdate(
+                    newRuns: [new Run(Start + 1, Length - 1)],
+                    removeRuns: [this]
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(
+                newRuns: [new Run(index + 1, newRunLength)],
+                removeRuns: Array.Empty<Run>()
+            );
+        }
+
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+
+        public override string ToString()
+        {
+            return $"[{Start},{Length}]";
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is Run run &&
+                   Start == run.Start &&
+                   Length == run.Length;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Start, Length);
+        }
+
+        public static bool operator ==(Run? obj1, Run? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2);
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+
+    public partial class IndexSet
+    {
+        private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+        public IndexSet()
+        {
+        }
+
+        public IndexSet(int[] indices)
+        {
+            foreach (var i in indices) Set(i);
+        }
+
+        public static IndexSet FromRunLengthEncoded(int[] rle)
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < rle.Length; i += 2)
+            {
+                var start = rle[i];
+                var length = rle[i + 1];
+                set.runs.Add(start, new Run(start, length));
+            }
+
+            return set;
+        }
+
+        public bool IsSet(int index)
+        {
+            if (runs.ContainsKey(index)) return true;
+
+            var run = GetRunAt(index);
+            if (run == null) return false;
+            return true;
+        }
+
+        public void Set(int index)
+        {
+            if (IsSet(index)) return;
+
+            var runBefore = GetRunAt(index - 1);
+            var runAfter = GetRunExact(index + 1);
+
+            if (runBefore == null)
+            {
+                if (runAfter == null)
+                {
+                    CreateNewRun(index);
+                }
+                else
+                {
+                    HandleUpdate(runAfter.ExpandToInclude(index));
+                }
+            }
+            else
+            {
+                if (runAfter == null)
+                {
+                    HandleUpdate(runBefore.ExpandToInclude(index));
+                }
+                else
+                {
+                    // new index will connect runBefore with runAfter. We merge!
+                    HandleUpdate(new RunUpdate(
+                        newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)],
+                        removeRuns: [runBefore, runAfter]
+                    ));
+                }
+            }
+        }
+
+        public void Unset(int index)
+        {
+            if (runs.ContainsKey(index))
+            {
+                HandleUpdate(runs[index].Unset(index));
+            }
+            else
+            {
+                var run = GetRunAt(index);
+                if (run == null) return;
+                HandleUpdate(run.Unset(index));
+            }
+        }
+
+        public void Iterate(Action<int> onIndex)
+        {
+            foreach (var run in runs.Values)
+            {
+                run.Iterate(onIndex);
+            }
+        }
+
+        public int[] RunLengthEncoded()
+        {
+            return Encode().ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join("&", runs.Select(r => r.ToString()).ToArray());
+        }
+
+        private IEnumerable<int> Encode()
+        {
+            foreach (var pair in runs)
+            {
+                yield return pair.Value.Start;
+                yield return pair.Value.Length;
+            }
+        }
+
+        private Run? GetRunAt(int index)
+        {
+            foreach (var run in runs.Values)
+            {
+                if (run.Includes(index)) return run;
+            }
+            return null;
+        }
+
+        private Run? GetRunExact(int index)
+        {
+            if (runs.ContainsKey(index)) return runs[index];
+            return null;
+        }
+
+        private void HandleUpdate(RunUpdate runUpdate)
+        {
+            foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+        }
+
+        private void CreateNewRun(int index)
+        {
+            if (runs.ContainsKey(index + 1))
+            {
+                var length = runs[index + 1].Length + 1;
+                runs.Add(index, new Run(index, length));
+                runs.Remove(index + 1);
+            }
+            else
+            {
+                runs.Add(index, new Run(index, 1));
+            }
+        }
+    }
+
+    public partial class IndexSet
+    {
+        public IndexSet Overlap(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public IndexSet Merge(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(result.Set);
+            other.Iterate(result.Set);
+            return result;
+        }
+
+        public IndexSet Without(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (!other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (obj is IndexSet set)
+            {
+                if (set.runs.Count != runs.Count) return false;
+                foreach (var pair in runs)
+                {
+                    if (!set.runs.ContainsKey(pair.Key)) return false;
+                    if (set.runs[pair.Key] != pair.Value) return false;
+                }
+                return true;
+            }
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(runs);
+        }
+
+        public static bool operator ==(IndexSet? obj1, IndexSet? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2);
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs

@@ -0,0 +1,94 @@
+using NUnit.Framework;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingLogicalTests
+    {
+        [Test]
+        public void EqualityTest()
+        {
+            var setA = new IndexSet([1, 2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.EqualTo(setB));
+            Assert.That(setA == setB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var setA = new IndexSet([1, 2, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var setA = new IndexSet([1, 2, 3]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest3()
+        {
+            var setA = new IndexSet([2, 3, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest()
+        {
+            var setA = new IndexSet([2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void Overlap()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([3, 4, 5, 11]);
+
+            var set = setA.Overlap(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Merge()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]);
+
+            var set = setA.Merge(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Without()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 14]);
+
+            var set = setA.Without(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -1,12 +1,40 @@
 using NUnit.Framework;
-using NUnit.Framework.Interfaces;
-using static FrameworkTests.Utils.RunLengthEncodingTests;
 
 namespace FrameworkTests.Utils
 {
     [TestFixture]
     public class RunLengthEncodingRunTests
     {
+        [Test]
+        public void EqualityTest()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 4);
+
+            Assert.That(runA, Is.EqualTo(runB));
+            Assert.That(runA == runB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 5);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(2, 4);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
         [Test]
         [Combinatorial]
         public void RunIncludes(
@@ -33,23 +61,58 @@ namespace FrameworkTests.Utils
         }
 
         [Test]
-        public void RunExpandToInclude()
+        public void RunExpandThrowsWhenIndexNotAdjacent()
         {
             var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
             Assert.That(run.Includes(2));
             Assert.That(run.Includes(4));
             Assert.That(!run.Includes(5));
 
-            Assert.That(run.ExpandToInclude(1), Is.False);
-            Assert.That(run.ExpandToInclude(2), Is.False);
-            Assert.That(run.ExpandToInclude(4), Is.False);
-            Assert.That(run.ExpandToInclude(6), Is.False);
+            Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf<Exception>());
+        }
 
-            Assert.That(run.ExpandToInclude(5), Is.True);
+        [Test]
+        public void RunExpandThrowsWhenIndexAlreadyIncluded()
+        {
+            var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf<Exception>());
+        }
+
+        [Test]
+        public void RunExpandToIncludeAfter()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(5);
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
             Assert.That(run.Includes(5));
             Assert.That(!run.Includes(6));
         }
 
+        [Test]
+        public void RunExpandToIncludeBefore()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(1);
+
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
         [Test]
         public void RunCanUnsetLastIndex()
         {
@@ -100,94 +163,9 @@ namespace FrameworkTests.Utils
         {
             var run = new Run(2, 4);
             var seen = new List<int>();
-            run.Iterate(i => seen.Add(i));
+            run.Iterate(seen.Add);
 
             CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
         }
     }
-
-    public class Run
-    {
-        public Run(int start, int length)
-        {
-            Start = start;
-            Length = length;
-        }
-
-        public int Start { get; }
-        public int Length { get; private set; }
-
-        public bool Includes(int index)
-        {
-            return index >= Start && index < (Start + Length);
-        }
-
-        public bool ExpandToInclude(int index)
-        {
-            if (index == (Start + Length))
-            {
-                Length++;
-                return true;
-            }
-            return false;
-        }
-
-        public RunUpdate Unset(int index)
-        {
-            if (!Includes(index))
-            {
-                return new RunUpdate();
-            }
-
-            if (index == Start)
-            {
-                // First index: Replace self with new run at next index, unless empty.
-                if (Length == 1)
-                {
-                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
-                }
-                return new RunUpdate(
-                    newRuns: new[] { new Run(Start + 1, Length - 1) },
-                    removeRuns: new[] { this }
-                );
-            }
-
-            if (index == (Start + Length - 1))
-            {
-                // Last index: Become one smaller.
-                Length--;
-                return new RunUpdate();
-            }
-
-            // Split:
-            var newRunLength = (Start + Length - 1) - index;
-            Length = index - Start;
-            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
-        }
-        
-        public void Iterate(Action<int> action)
-        {
-            for (var i = 0; i < Length; i++)
-            {
-                action(Start + i);
-            }
-        }
-    }
-
-    public class RunUpdate
-    {
-        public RunUpdate()
-            : this(Array.Empty<Run>(), Array.Empty<Run>())
-        {
-        }
-
-        public RunUpdate(Run[] newRuns, Run[] removeRuns)
-        {
-            NewRuns = newRuns;
-            RemoveRuns = removeRuns;
-        }
-
-        public Run[] NewRuns { get; }
-        public Run[] RemoveRuns { get; }
-    }
 }

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -1,9 +1,4 @@
-using Logging;
-using Microsoft.VisualStudio.TestPlatform.Common;
-using NuGet.Frameworks;
-using NUnit.Framework;
-using System.Collections.Concurrent;
-using System.Numerics;
+using NUnit.Framework;
 using Utils;
 
 namespace FrameworkTests.Utils
@@ -119,6 +114,19 @@ namespace FrameworkTests.Utils
             }, encoded);
         }
 
+        [Test]
+        public void SetIndexBetweenRuns()
+        {
+            var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                8, 7
+            }, encoded);
+        }
+
         [Test]
         public void SetIndexAfterRun()
         {
@@ -201,120 +209,5 @@ namespace FrameworkTests.Utils
             all.Sort();
             return all.ToArray();
         }
-
-        public class IndexSet
-        {
-            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
-
-            public IndexSet()
-            {
-            }
-
-            public IndexSet(int[] indices)
-            {
-                foreach (var i in indices) Set(i);
-            }
-
-            public static IndexSet FromRunLengthEncoded(int[] rle)
-            {
-                var set = new IndexSet();
-                for (var i = 0; i < rle.Length; i += 2)
-                {
-                    var start = rle[i];
-                    var length = rle[i + 1];
-                    set.runs.Add(start, new Run(start, length));
-                }
-
-                return set;
-            }
-
-            public bool IsSet(int index)
-            {
-                if (runs.ContainsKey(index)) return true;
-
-                var run = GetRunBefore(index);
-                if (run == null) return false;
-
-                return run.Includes(index);
-            }
-
-            public void Set(int index)
-            {
-                if (runs.ContainsKey(index)) return;
-
-                var run = GetRunBefore(index);
-                if (run == null || !run.ExpandToInclude(index))
-                {
-                    CreateNewRun(index);
-                }
-            }
-
-            public void Unset(int index)
-            {
-                if (runs.ContainsKey(index))
-                {
-                    HandleUpdate(runs[index].Unset(index));
-                }
-                else
-                {
-                    var run = GetRunBefore(index);
-                    if (run == null) return;
-                    HandleUpdate(run.Unset(index));
-                }
-            }
-
-            public void Iterate(Action<int> onIndex)
-            {
-                foreach (var run in runs.Values)
-                {
-                    run.Iterate(onIndex);
-                }
-            }
-
-            public int[] RunLengthEncoded()
-            {
-                return Encode().ToArray();
-            }
-
-            private IEnumerable<int> Encode()
-            {
-                foreach (var pair in runs)
-                {
-                    yield return pair.Value.Start;
-                    yield return pair.Value.Length;
-                }
-            }
-
-            private Run? GetRunBefore(int index)
-            {
-                Run? result = null;
-                foreach (var pair in runs)
-                {
-                    if (pair.Key < index) result = pair.Value;
-                    else return result;
-                }
-                return result;
-            }
-
-            private void HandleUpdate(RunUpdate runUpdate)
-            {
-                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
-                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
-            }
-
-            private void CreateNewRun(int index)
-            {
-                if (runs.ContainsKey(index + 1))
-                {
-                    var length = runs[index + 1].Length + 1;
-                    runs.Add(index, new Run(index, length));
-                    runs.Remove(index + 1);
-                }
-                else
-                {
-                    runs.Add(index, new Run(index, 1));
-                }
-            }
-        }
     }
 }

Tools/AutoClient/App.cs

@@ -0,0 +1,40 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class App
+    {
+        public App(Configuration config)
+        {
+            Config = config;
+
+            Log = new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "autoclient")),
+                new ConsoleLog()
+            );
+
+            Generator = CreateGenerator();
+            CidRepo = new CidRepo(config);
+            Performance = new Performance(new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "performance")),
+                new ConsoleLog()
+            ));
+        }
+
+        public Configuration Config { get; }
+        public ILog Log { get; }
+        public IFileGenerator Generator { get; }
+        public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
+        public CidRepo CidRepo { get; }
+        public Performance Performance { get; }
+
+        private IFileGenerator CreateGenerator()
+        {
+            if (Config.FileSizeMb > 0)
+            {
+                return new RandomFileGenerator(Config, Log);
+            }
+            return new ImageGenerator(Log);
+        }
+    }
+}

Tools/AutoClient/AutoClient.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/AutoClient/CidRepo.cs

@@ -0,0 +1,85 @@
+namespace AutoClient
+{
+    public class CidRepo
+    {
+        private readonly Random random = new Random();
+        private readonly object _lock = new object();
+        private readonly List<CidEntry> entries = new List<CidEntry>();
+        private readonly Configuration config;
+
+        public CidRepo(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Add(string nodeId, string cid, long knownSize)
+        {
+            lock (_lock)
+            {
+                entries.Add(new CidEntry(nodeId, cid, knownSize));
+            }
+        }
+
+        public void AddEncoded(string originalCid, string encodedCid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == originalCid);
+                if (entry == null) return;
+
+                entry.Encoded = encodedCid;
+            }
+        }
+
+        public string? GetForeignCid(string myNodeId)
+        {
+            lock (_lock)
+            {
+                while (true)
+                {
+                    if (!entries.Any()) return null;
+                    var available = entries.Where(e => e.NodeId != myNodeId).ToArray();
+                    if (!available.Any()) return null;
+
+                    var i = random.Next(0, available.Length);
+                    var entry = available[i];
+
+                    if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes)))
+                    {
+                        entries.Remove(entry);
+                    }
+                    else
+                    {
+                        return entry.Cid;
+                    }
+                }
+            }
+        }
+
+        public long? GetSizeForCid(string cid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == cid);
+                if (entry == null) return null;
+                return entry.KnownSize;
+            }
+        }
+    }
+
+    public class CidEntry
+    {
+        public CidEntry(string nodeId, string cid, long knownSize)
+        {
+            NodeId = nodeId;
+            Cid = cid;
+            KnownSize = knownSize;
+        }
+
+        public string NodeId { get; }
+        public string Cid { get; }
+        public string Encoded { get; set; } = string.Empty;
+        public long KnownSize { get; }
+        public DateTime CreatedUtc { get; } = DateTime.UtcNow;
+    }
+}

Tools/AutoClient/CodexUser.cs

@@ -0,0 +1,57 @@
+using CodexOpenApi;
+using Logging;
+using Utils;
+
+namespace AutoClient
+{
+    public class CodexUser
+    {
+        private readonly App app;
+        private readonly CodexApi codex;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly List<Purchaser> purchasers = new List<Purchaser>();
+        private Task starterTask = Task.CompletedTask;
+        private readonly string nodeId = Guid.NewGuid().ToString();
+
+        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            this.app = app;
+            this.codex = codex;
+            this.client = client;
+            this.address = address;
+        }
+
+        public void Start(int index)
+        {
+            for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
+            {
+                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+            }
+
+            var delayPerPurchaser =
+                TimeSpan.FromSeconds(10 * index) +
+                TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases;
+
+            starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser));
+        }
+
+        private async Task StartPurchasers(TimeSpan delayPerPurchaser)
+        {
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Start();
+                await Task.Delay(delayPerPurchaser);
+            }
+        }
+
+        public void Stop()
+        {
+            starterTask.Wait();
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Stop();
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -4,11 +4,8 @@ namespace AutoClient
 {
     public class Configuration
     {
-        [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
-        public string CodexHost { get; set; } = "http://localhost";
-
-        [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
-        public int CodexPort { get; set; } = 8080;
+        [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
+        public string CodexEndpoints { get; set; } = "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -22,11 +19,11 @@ namespace AutoClient
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
         public int ContractExpiryMinutes { get; set; } = 15;
 
-        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
-        public int NumHosts { get; set; } = 5;
+        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
+        public int NumHosts { get; set; } = 10;
 
-        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
-        public int HostTolerance { get; set; } = 2;
+        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
+        public int HostTolerance { get; set; } = 5;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
         public int Price { get; set; } = 10;

Tools/AutoClient/ImageGenerator.cs

@@ -11,16 +11,16 @@ namespace AutoClient
 
     public class ImageGenerator : IFileGenerator
     {
-        private LogSplitter log;
+        private readonly ILog log;
 
-        public ImageGenerator(LogSplitter log)
+        public ImageGenerator(ILog log)
         {
             this.log = log;
         }
 
         public async Task<string> Generate()
         {
-            log.Log("Fetching random image from picsum.photos...");
+            log.Debug("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 

Tools/AutoClient/Performance.cs

@@ -0,0 +1,65 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class Performance
+    {
+        private readonly ILog log;
+
+        public Performance(ILog log)
+        {
+            this.log = log;
+        }
+
+        public void DownloadFailed(Exception ex)
+        {
+            Log($"Download failed: {ex}");
+        }
+
+        public void DownloadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Download successful: {bytesPerSecond} bytes per second");
+        }
+
+        public void StorageContractCancelled()
+        {
+            Log("Contract cancelled");
+        }
+
+        public void StorageContractErrored(string error)
+        {
+            Log($"Contract errored: {error}");
+        }
+
+        public void StorageContractFinished()
+        {
+            Log("Contract finished");
+        }
+
+        public void StorageContractStarted()
+        {
+            Log("Contract started");
+        }
+
+        public void UploadFailed(Exception ex)
+        {
+            Log($"Upload failed: {ex}");
+        }
+
+        public void UploadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Upload successful: {bytesPerSecond} bytes per second");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

Tools/AutoClient/Program.cs

@@ -1,16 +1,20 @@
 using ArgsUniform;
 using AutoClient;
 using CodexOpenApi;
-using Core;
-using Logging;
 using Utils;
 
-public static class Program
+public class Program
 {
+    private readonly App app;
+
+    public Program(Configuration config)
+    {
+        app = new App(config);
+    }
+
     public static async Task Main(string[] args)
     {
         var cts = new CancellationTokenSource();
-        var cancellationToken = cts.Token;
         Console.CancelKeyPress += (sender, args) => cts.Cancel();
 
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
@@ -21,58 +25,70 @@ public static class Program
             throw new Exception("Number of concurrent purchases must be > 0");
         }
 
-        var log = new LogSplitter(
-            new FileLog(Path.Combine(config.LogPath, "autoclient")),
-            new ConsoleLog()
-        );
+        var p = new Program(config);
+        await p.Run();
+    }
+
+    public async Task Run()
+    {
+        var codexUsers = await CreateUsers();
+
+        var i = 0;
+        foreach (var user in codexUsers)
+        {
+            user.Start(i);
+            i++;
+        }
+
+        app.Cts.Token.WaitHandle.WaitOne();
+
+        foreach (var user in codexUsers) user.Stop();
+
+        app.Log.Log("Done");
+    }
+
+    private async Task<CodexUser[]> CreateUsers()
+    {
+        var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
+        var result = new List<CodexUser>();
+
+        foreach (var e in endpointStrs)
+        {
+            result.Add(await CreateUser(e));
+        }
+
+        return result.ToArray();
+    }
+
+    private async Task<CodexUser> CreateUser(string endpoint)
+    {
+        var splitIndex = endpoint.LastIndexOf(':');
+        var host = endpoint.Substring(0, splitIndex);
+        var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
 
         var address = new Address(
-            host: config.CodexHost,
-            port: config.CodexPort
+            host: host,
+            port: port
         );
 
-        log.Log($"Start. Address: {address}");
-
-        var generator = CreateGenerator(config, log);
-
         var client = new HttpClient();
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
-        await CheckCodex(codex, log);
+        app.Log.Log($"Checking Codex at {address}...");
+        await CheckCodex(codex);
+        app.Log.Log("OK");
 
-        var purchasers = new List<Purchaser>();
-        for (var i = 0; i < config.NumConcurrentPurchases; i++)
-        {
-            purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
-            );
-        }
-
-        var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
-        foreach (var purchaser in purchasers) 
-        {
-            purchaser.Start();
-            await Task.Delay(delayPerPurchaser);
-        }
-
-        cancellationToken.WaitHandle.WaitOne();
-
-        log.Log("Done.");
+        return new CodexUser(
+            app,
+            codex,
+            client,
+            address
+        );
     }
 
-    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    private async Task CheckCodex(CodexApi codex)
     {
-        if (config.FileSizeMb > 0)
-        {
-            return new RandomFileGenerator(config, log);
-        }
-        return new ImageGenerator(log);
-    }
-
-    private static async Task CheckCodex(CodexApi codex, ILog log)
-    {
-        log.Log("Checking Codex...");
         try
         {
             var info = await codex.GetDebugInfoAsync();
@@ -80,7 +96,7 @@ public static class Program
         }
         catch (Exception ex)
         {
-            log.Log($"Codex not OK: {ex}");
+            app.Log.Error($"Codex not OK: {ex}");
             throw;
         }
     }

Tools/AutoClient/Purchaser.cs

@@ -8,114 +8,197 @@ namespace AutoClient
 {
     public class Purchaser
     {
+        private readonly App app;
+        private readonly string nodeId;
         private readonly ILog log;
         private readonly HttpClient client;
         private readonly Address address;
         private readonly CodexApi codex;
-        private readonly Configuration config;
-        private readonly IFileGenerator generator;
-        private readonly CancellationToken ct;
+        private Task workerTask = Task.CompletedTask;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
+        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
         {
+            this.app = app;
+            this.nodeId = nodeId;
             this.log = log;
             this.client = client;
             this.address = address;
             this.codex = codex;
-            this.config = config;
-            this.generator = generator;
-            this.ct = ct;
         }
 
         public void Start()
         {
-            Task.Run(Worker);
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
         }
 
         private async Task Worker()
         {
-            while (!ct.IsCancellationRequested)
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
             {
-                var pid = await StartNewPurchase();
-                await WaitTillFinished(pid);
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(nodeId);
+            if (cid == null) return;
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+                {
+                    using var fileStream = File.OpenWrite(filename);
+                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
+                    fileResponse.Stream.CopyTo(fileStream);
+                }
+                var time = sw.Elapsed;
+                File.Delete(filename);
+                app.Performance.DownloadSuccessful(size.Value, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
             }
         }
 
         private async Task<string> StartNewPurchase()
         {
             var file = await CreateFile();
-            var cid = await UploadFile(file);
-            return await RequestStorage(cid);
+            try
+            {
+                var cid = await UploadFile(file);
+                return await RequestStorage(cid);
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
         }
 
         private async Task<string> CreateFile()
         {
-            return await generator.Generate();
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
         }
 
         private async Task<ContentId> UploadFile(string filename)
         {
-            // Copied from CodexNode :/
             using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(nodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
 
-            log.Log($"Uploading file {filename}...");
-            var response = await codex.UploadAsync(fileStream, ct);
+        private async Task<ContentId> UploadStream(FileStream fileStream)
+        {
+            log.Debug($"Uploading file...");
+            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
 
-            log.Log($"Uploaded file. Received contentId: '{response}'.");
+            log.Debug($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
         private async Task<string> RequestStorage(ContentId cid)
         {
-            log.Log("Requesting storage for " + cid.Id);
+            log.Debug("Requesting storage for " + cid.Id);
             var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
             {
-                Collateral = config.RequiredCollateral.ToString(),
-                Duration = (config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = config.NumHosts,
-                Reward = config.Price.ToString(),
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
                 ProofProbability = "15",
-                Tolerance = config.HostTolerance
-            }, ct);
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
 
-            log.Log("Purchase ID: " + result);
+            log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
 
             return result;
         }
 
-        private async Task<string?> GetPurchaseState(string pid)
+        private async Task<string> GetEncodedCid(string pid)
         {
             try
             {
-                // openapi still don't match code.
-                var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-                if (string.IsNullOrEmpty(str)) return null;
-                var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                log.Log($"Purchase {pid} is {sp.State}");
-                if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
-                return sp.State;
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
             }
-            catch
+            catch (Exception ex)
             {
-                return null;
+                log.Error(ex.ToString());
+                throw;
             }
         }
 
+        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
         private async Task WaitTillFinished(string pid)
         {
-            log.Log("Waiting...");
             try
             {
                 var emptyResponseTolerance = 10;
-                while (true)
+                while (!app.Cts.Token.IsCancellationRequested)
                 {
-                    var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
-                    if (string.IsNullOrEmpty(status))
+                    var purchase = await GetStoragePurchase(pid);
+                    if (purchase == null)
                     {
+                        await FixedShortDelay();
                         emptyResponseTolerance--;
                         if (emptyResponseTolerance == 0)
                         {
@@ -123,19 +206,28 @@ namespace AutoClient
                             await ExpiryTimeDelay();
                             return;
                         }
+                        continue;
                     }
-                    else
+                    var status = purchase.State.ToLowerInvariant();
+                    if (status.Contains("cancel"))
                     {
-                        if (status.Contains("cancel") ||
-                            status.Contains("error") ||
-                            status.Contains("finished"))
-                        {
-                            return;
-                        }
-                        if (status.Contains("started"))
-                        {
-                            await FixedDurationDelay();
-                        }
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (status.Contains("error"))
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (status.Contains("finished"))
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (status.Contains("started"))
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
                     }
 
                     await FixedShortDelay();
@@ -150,17 +242,17 @@ namespace AutoClient
 
         private async Task FixedDurationDelay()
         {
-            await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task ExpiryTimeDelay()
         {
-            await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task FixedShortDelay()
         {
-            await Task.Delay(15 * 1000, ct);
+            await Task.Delay(15 * 1000, app.Cts.Token);
         }
     }
 }

Tools/AutoClient/README.md

@@ -6,7 +6,7 @@ Can generate random images or random data of a specified size.
 
 ## How to run
 
-- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- dotnet 8.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
 - docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
 
 ## Configuration options

Tools/AutoClient/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/BiblioTech/AdminChecker.cs

@@ -1,5 +1,7 @@
-using Discord;
+using BiblioTech.Options;
+using Discord;
 using Discord.WebSocket;
+using Org.BouncyCastle.Utilities;
 
 namespace BiblioTech
 {
@@ -27,9 +29,21 @@ namespace BiblioTech
             return channel.Id == Program.Config.AdminChannelId;
         }
 
-        public ISocketMessageChannel GetAdminChannel()
+        public async Task SendInAdminChannel(string msg)
         {
-            return adminChannel;
+            await SendInAdminChannel(msg.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        public async Task SendInAdminChannel(string[] lines)
+        {
+            var chunker = new LineChunker(lines);
+            var chunks = chunker.GetChunks();
+            if (!chunks.Any()) return;
+
+            foreach (var chunk in chunks)
+            {
+                await adminChannel.SendMessageAsync(string.Join(Environment.NewLine, chunk));
+            }
         }
 
         public void SetAdminChannel(ISocketMessageChannel adminChannel)

Tools/BiblioTech/BaseCommand.cs

@@ -25,16 +25,13 @@ namespace BiblioTech
             catch (Exception ex)
             {
                 var msg = "Failed with exception: " + ex;
-                if (IsInAdminChannel(command))
-                {
-                    await command.FollowupAsync(msg.Substring(0, Math.Min(1900, msg.Length)));
-                }
-                else
-                {
-                    await command.FollowupAsync("Something failed while trying to do that...", ephemeral: true);
-                    await Program.AdminChecker.GetAdminChannel().SendMessageAsync(msg);
-                }
                 Program.Log.Error(msg);
+
+                if (!IsInAdminChannel(command))
+                {
+                    await command.FollowupAsync("Something failed while trying to do that... (error details posted in admin channel)", ephemeral: true);
+                }
+                await Program.AdminChecker.SendInAdminChannel(msg);
             }
         }
 
@@ -62,5 +59,20 @@ namespace BiblioTech
             if (IsSenderAdmin(context.Command) && targetUser != null) return targetUser;
             return context.Command.User;
         }
+
+        protected string Mention(SocketUser user)
+        {
+            return Mention(user.Id);
+        }
+
+        protected string Mention(IUser user)
+        {
+            return Mention(user.Id);
+        }
+
+        protected string Mention(ulong userId)
+        {
+            return $"<@{userId}>";
+        }
     }
 }

Tools/BiblioTech/BiblioTech.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -12,6 +12,7 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CodexCidChecker.cs

@@ -0,0 +1,198 @@
+using CodexOpenApi;
+using IdentityModel.Client;
+using Utils;
+
+namespace BiblioTech
+{
+    public class CodexCidChecker
+    {
+        private static readonly string nl = Environment.NewLine;
+        private readonly Configuration config;
+        private CodexApi? currentCodexNode;
+
+        public CodexCidChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public async Task<CheckResponse> PerformCheck(string cid)
+        {
+            if (string.IsNullOrEmpty(config.CodexEndpoint))
+            {
+                return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
+            }
+
+            try
+            {
+                var codex = GetCodex();
+                var nodeCheck = await CheckCodex(codex);
+                if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
+
+                return await PerformCheck(codex, cid);
+            }
+            catch (Exception ex)
+            {
+                return new CheckResponse(false, "Internal server error", ex.ToString());
+            }
+        }
+
+        private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
+        {
+            try
+            {
+                var manifest = await codex.DownloadNetworkManifestAsync(cid);
+                return SuccessMessage(manifest);
+            }
+            catch (ApiException apiEx)
+            {
+                if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
+                if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
+                return UnexpectedReturnCode(apiEx.Response);
+            }
+            catch (Exception ex)
+            {
+                return UnexpectedException(ex);
+            }
+        }
+
+        #region Response formatting
+
+        private CheckResponse SuccessMessage(DataItem content)
+        {
+            return FormatResponse(
+                success: true,
+                title: $"Success: '{content.Cid}'",
+                error: "",
+                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"blockSize: {content.Manifest.BlockSize} bytes",
+                $"protected: {content.Manifest.Protected}"
+            );
+        }
+
+        private CheckResponse UnexpectedException(Exception ex)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Unexpected error",
+                error: ex.ToString(),
+                content: "Details will be sent to the bot-admin channel."
+            );
+        }
+
+        private CheckResponse UnexpectedReturnCode(string response)
+        {
+            var msg = "Unexpected return code. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Unexpected return code",
+                error: msg,
+                content: msg
+            );
+        }
+
+        private CheckResponse FailedToFetch(string response)
+        {
+            var msg = "Failed to download content. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Could not download content",
+                error: msg,
+                msg,
+                $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
+            );
+        }
+
+        private CheckResponse CidFormatInvalid(string response)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Invalid format",
+                error: "",
+                content: "Provided CID is not formatted correctly."
+            );
+        }
+
+        private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
+        {
+            var msg = string.Join(nl,
+                new string[]
+                {
+                    title,
+                    "```"
+                }
+                .Concat(content)
+                .Concat(new string[]
+                {
+                    "```"
+                })
+            ) + nl + nl;
+
+            return new CheckResponse(success, msg, error);
+        }
+
+        #endregion
+
+        #region Codex Node API 
+
+        private CodexApi GetCodex()
+        {
+            if (currentCodexNode == null) currentCodexNode = CreateCodex();
+            return currentCodexNode;
+        }
+
+        private async Task<bool> CheckCodex(CodexApi codex)
+        {
+            try
+            {
+                var info = await currentCodexNode!.GetDebugInfoAsync();
+                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
+                return true;
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
+        private CodexApi CreateCodex()
+        {
+            var endpoint = config.CodexEndpoint;
+            var splitIndex = endpoint.LastIndexOf(':');
+            var host = endpoint.Substring(0, splitIndex);
+            var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
+
+            var address = new Address(
+                host: host,
+                port: port
+            );
+
+            var client = new HttpClient();
+            if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
+            {
+                var tokens = config.CodexEndpointAuth.Split(':');
+                if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
+                client.SetBasicAuthentication(tokens[0], tokens[1]);    
+            }
+
+            var codex = new CodexApi(client);
+            codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return codex;
+        }
+
+        #endregion
+    }
+
+    public class CheckResponse
+    {
+        public CheckResponse(bool success, string message, string error)
+        {
+            Success = success;
+            Message = message;
+            Error = error;
+        }
+
+        public bool Success { get; }
+        public string Message { get; }
+        public string Error { get; }
+    }
+}

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -0,0 +1,38 @@
+using BiblioTech.Options;
+
+namespace BiblioTech.Commands
+{
+    public class CheckCidCommand : BaseCommand
+    {
+        private readonly StringOption cidOption = new StringOption(
+            name: "cid",
+            description: "Codex Content-Identifier",
+            isRequired: true);
+        private readonly CodexCidChecker checker;
+
+        public CheckCidCommand(CodexCidChecker checker)
+        {
+            this.checker = checker;
+        }
+
+        public override string Name => "check";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Checks if content is available in the testnet.";
+        public override CommandOption[] Options => new[] { cidOption };
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            var user = context.Command.User;
+            var cid = await cidOption.Parse(context);
+            if (string.IsNullOrEmpty(cid))
+            {
+                await context.Followup("Option 'cid' was not received.");
+                return;
+            }
+
+            var response = await checker.PerformCheck(cid);
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
+            await context.Followup(response.Message);
+        }
+    }
+}

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -28,6 +28,7 @@ namespace BiblioTech.Commands
             if (addr == null)
             {
                 await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' but address has not been set.");
                 return;
             }
 

Tools/BiblioTech/Commands/MintCommand.cs

@@ -28,6 +28,7 @@ namespace BiblioTech.Commands
             if (addr == null)
             {
                 await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' but address has not been set.");
                 return;
             }
 
@@ -42,9 +43,17 @@ namespace BiblioTech.Commands
                 mintedTokens = ProcessTokens(contracts, addr, report);
             });
 
+            var reportLine = string.Join(Environment.NewLine, report);
             Program.UserRepo.AddMintEventForUser(userId, addr, sentEth, mintedTokens);
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' successfully. ({reportLine})");
 
-            await context.Followup(string.Join(Environment.NewLine, report));
+            await context.Followup(reportLine);
+        }
+
+        private string Format<T>(Transaction<T>? transaction)
+        {
+            if (transaction == null) return "-";
+            return transaction.ToString();
         }
 
         private Transaction<TestToken>? ProcessTokens(ICodexContracts contracts, EthAddress addr, List<string> report)

Tools/BiblioTech/Commands/UserAssociateCommand.cs

@@ -1,4 +1,8 @@
 using BiblioTech.Options;
+using Discord;
+using GethPlugin;
+using k8s.KubeConfigModels;
+using NBitcoin.Secp256k1;
 
 namespace BiblioTech.Commands
 {
@@ -23,30 +27,56 @@ namespace BiblioTech.Commands
         protected override async Task Invoke(CommandContext context)
         {
             var user = GetUserFromCommand(optionalUser, context);
-            var data = await ethOption.Parse(context);
-            if (data == null) return;
+            var newAddress = await ethOption.Parse(context);
+            if (newAddress == null) return;
 
             var currentAddress = Program.UserRepo.GetCurrentAddressForUser(user);
             if (currentAddress != null && !IsSenderAdmin(context.Command))
             {
                 await context.Followup($"You've already set your Ethereum address to {currentAddress}.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but already has an address set. ({currentAddress})");
                 return;
             }
 
-            var result = Program.UserRepo.AssociateUserWithAddress(user, data);
-            if (result)
+            var result = Program.UserRepo.AssociateUserWithAddress(user, newAddress);
+            switch (result)
             {
-                await context.Followup(new string[]
-                {
+                case SetAddressResponse.OK:
+                    await ResponseOK(context, user, newAddress);
+                    break;
+                case SetAddressResponse.AddressAlreadyInUse:
+                    await ResponseAlreadyUsed(context, user, newAddress);
+                    break;
+                case SetAddressResponse.CreateUserFailed:
+                    await ResponseCreateUserFailed(context, user);
+                    break;
+                default:
+                    throw new Exception("Unknown SetAddressResponse mode");
+            }
+        }
+
+        private async Task ResponseCreateUserFailed(CommandContext context, IUser user)
+        {
+            await context.Followup("Internal error. Error details sent to admin.");
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but failed to create new user.");
+        }
+
+        private async Task ResponseAlreadyUsed(CommandContext context, IUser user, EthAddress newAddress)
+        {
+            await context.Followup("This address is already in use by another user.");
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but the provided address is already in use by another user. (address: {newAddress})");
+        }
+
+        private async Task ResponseOK(CommandContext context, IUser user, GethPlugin.EthAddress newAddress)
+        {
+            await context.Followup(new string[]
+{
                     "Done! Thank you for joining the test net!",
-                    "By default, the bot will @-mention you with test-net reward related notifications.",
+                    "By default, the bot will @-mention you with test-net related notifications.",
                     $"You can enable/disable this behavior with the '/{notifyCommand.Name}' command."
-                });
-            }
-            else
-            {
-                await context.Followup("That didn't work.");
-            }
+});
+
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' successfully. ({newAddress})");
         }
     }
 }