Merge branch 'master' into feature/block-retransmit

# Conflicts:
#	ProjectPlugins/CodexPlugin/ApiChecker.cs

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -5,6 +5,16 @@
         private static readonly Random random = new Random();
         private static readonly object @lock = new object();
 
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
+
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
             lock (@lock)

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -66,7 +69,11 @@ namespace CodexContractsPlugin.ChainMonitor
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
             log.Log($"ChainState updating: {events.BlockInterval}");
 
@@ -109,7 +116,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -117,7 +124,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -125,7 +132,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -133,7 +140,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -142,13 +149,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -162,10 +177,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
 using NethereumWorkflow.BlockUtils;
 using Utils;
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "6B-94-24-A4-D5-01-6F-12-E9-34-74-36-80-57-7A-3A-79-8C-E8-02-68-B7-05-DA-50-A0-5C-B1-02-B9-AE-C6";
+        private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -73,7 +73,7 @@ namespace CodexPlugin
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -88,25 +88,25 @@ namespace CodexPlugin
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
         public StorageAvailability[] GetAvailabilities()
         {
-            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
             return mapper.Map(collection);
         }
 
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:0.1.4";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:0.1.6-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -83,33 +83,12 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
 
     DebugInfo:
       type: object
@@ -351,15 +330,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -477,9 +456,35 @@ paths:
 
   "/data/{cid}/network":
     get:
-      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
       tags: [ Data ]
       operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
+    get:
+      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      tags: [ Data ]
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -502,6 +507,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -747,7 +778,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -765,7 +796,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -7,4 +7,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,5 +1,7 @@
 using CodexPlugin;
+using FileUtils;
 using NUnit.Framework;
+using System.Diagnostics;
 using Utils;
 
 namespace CodexTests.BasicTests
@@ -10,11 +12,44 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
+        }
+
+        [Test]
+        public void InterruptUploadTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
         }
 
         private void PerformOneClientTest(ICodexNode primary)

Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs

@@ -0,0 +1,228 @@
+using CodexPlugin;
+using FileUtils;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class MultiswarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        public void Multiswarm(
+            [Values(3, 5)] int numFiles,
+            [Values(5, 20)] int fileSizeMb,
+            [Values(1)] int uploadersPerFile,
+            [Values(3)] int downloadersPerFile,
+            [Values(1)] int maxUploadsPerNode,
+            [Values(2, 3)] int maxDownloadsPerNode
+        )
+        {
+            var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            Assert.That(plan.NodePlans.Count, Is.LessThan(30));
+
+            RunThePlan(plan, fileSizeMb);
+        }
+
+        private void RunThePlan(Plan plan, int fileSizeMb)
+        {
+            foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB());
+            var nodes = StartCodex(plan.NodePlans.Count);
+            for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i];
+
+            // Upload all files to their nodes.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var uploader in filePlan.Uploaders)
+                {
+                    filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!);
+                }
+            }
+
+            Thread.Sleep(5000); // Everything is processed and announced.
+
+            // Start all downloads (almost) simultaneously.
+            var tasks = new List<Task>();
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloader in filePlan.Downloaders)
+                {
+                    tasks.Add(Task.Run(() =>
+                    {
+                        var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!);
+                        lock (filePlan.DownloadedFiles)
+                        {
+                            filePlan.DownloadedFiles.Add(downloadedFile);
+                        }
+                    }));
+                }
+            }
+
+            Task.WaitAll(tasks.ToArray());
+
+            // Assert all files are correct.
+            foreach (var filePlan in plan.FilePlans)
+            {
+                foreach (var downloadedFile in filePlan.DownloadedFiles)
+                {
+                    filePlan.File!.AssertIsEqual(downloadedFile);
+                }
+            }
+        }
+
+        private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
+            plan.Initialize();
+            plan.LogPlan(GetTestLog());
+            return plan;
+        }
+    }
+
+    public class FilePlan
+    {
+        public FilePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public TrackedFile? File { get; set; }
+        public ContentId? Cid { get; set; }
+        public List<TrackedFile?> DownloadedFiles { get; } = new List<TrackedFile?>();
+        public List<NodePlan> Uploaders { get; } = new List<NodePlan>();
+        public List<NodePlan> Downloaders { get; } = new List<NodePlan>();
+
+        public override string ToString()
+        {
+            return $"FilePlan[{Number}] " +
+                $"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " +
+                $"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class NodePlan
+    {
+        public NodePlan(int number)
+        {
+            Number = number;
+        }
+
+        public int Number { get; }
+        public ICodexNode? Node { get; set; }
+        public List<FilePlan> Uploads { get; } = new List<FilePlan>();
+        public List<FilePlan> Downloads { get; } = new List<FilePlan>();
+
+        public bool Contains(FilePlan plan)
+        {
+            return Uploads.Contains(plan) || Downloads.Contains(plan);
+        }
+
+        public override string ToString()
+        {
+            return $"NodePlan[{Number}] " +
+                $"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " +
+                $"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]";
+        }
+    }
+
+    public class Plan
+    {
+        private readonly int numFiles;
+        private readonly int uploadersPerFile;
+        private readonly int downloadersPerFile;
+        private readonly int maxUploadsPerNode;
+        private readonly int maxDownloadsPerNode;
+
+        public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
+        {
+            this.numFiles = numFiles;
+            this.uploadersPerFile = uploadersPerFile;
+            this.downloadersPerFile = downloadersPerFile;
+            this.maxUploadsPerNode = maxUploadsPerNode;
+            this.maxDownloadsPerNode = maxDownloadsPerNode;
+        }
+
+        public List<FilePlan> FilePlans { get; } = new List<FilePlan>();
+        public List<NodePlan> NodePlans { get; } = new List<NodePlan>();
+
+        public void Initialize()
+        {
+            for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i));
+            foreach (var filePlan in FilePlans)
+            {
+                while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan);
+                while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan);
+            }
+
+            CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number));
+            CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number));
+
+            foreach (var filePlan in FilePlans)
+            {
+                Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile));
+                Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile));
+            }
+            foreach (var nodePlan in NodePlans)
+            {
+                Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode));
+                Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode));
+            }
+        }
+
+        public void LogPlan(ILog log)
+        {
+            log.Log("The plan:");
+            log.Log("Input:");
+            log.Log($"numFiles: {numFiles}");
+            log.Log($"uploadersPerFile: {uploadersPerFile}");
+            log.Log($"downloadersPerFile: {downloadersPerFile}");
+            log.Log($"maxUploadsPerNode: {maxUploadsPerNode}");
+            log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}");
+            log.Log("Setup:");
+            log.Log($"number of nodes: {NodePlans.Count}");
+            foreach (var filePlan in FilePlans) log.Log(filePlan.ToString());
+            foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString());
+        }
+
+        private void AddDownloader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateDownloaderNode(filePlan);
+            filePlan.Downloaders.Add(nodePlan);
+            nodePlan.Downloads.Add(filePlan);
+        }
+
+        private void AddUploader(FilePlan filePlan)
+        {
+            var nodePlan = GetOrCreateUploaderNode(filePlan);
+            filePlan.Uploaders.Add(nodePlan);
+            nodePlan.Uploads.Add(filePlan);
+        }
+
+        private NodePlan GetOrCreateDownloaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+
+        private NodePlan GetOrCreateUploaderNode(FilePlan notIn)
+        {
+            var available = NodePlans.Where(n =>
+                n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
+            ).ToArray();
+            if (available.Any()) return RandomUtils.GetOneRandom(available);
+
+            var newNodePlan = new NodePlan(NodePlans.Count);
+            NodePlans.Add(newNodePlan);
+            return newNodePlan;
+        }
+    }
+}

Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs

@@ -0,0 +1,25 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class SwarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [CreateTranscript("swarm_retransmit")]
+        public void DetectBlockRetransmits(
+            [Values(1, 5, 10, 20)] int fileSize,
+            [Values(3, 5, 10, 20)] int numNodes
+        )
+        {
+            var nodes = StartCodex(numNodes);
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = nodes[0].UploadFile(file);
+
+            var tasks = nodes.Select(n => Task.Run(() => n.DownloadContent(cid))).ToArray();
+            Task.WaitAll(tasks);
+        }
+    }
+}

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests
         private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
         private readonly List<ulong> rewardsSeen = new List<ulong>();
         private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
-        private readonly List<string> receivedEvents = new List<string>();
+        private readonly List<ChainEventMessage> receivedEvents = new List<ChainEventMessage>();
  
         [Test]
         [DontDownloadLogs]
@@ -73,13 +73,18 @@ namespace CodexTests.UtilityTests
 
         private void AssertEventOccurance(string msg, int expectedCount)
         {
-            Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
+            Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount),
                 $"Event '{msg}' did not occure correct number of times.");
         }
 
         private void OnCommand(string timestamp, GiveRewardsCommand call)
         {
             Log($"<API call {timestamp}>");
+            foreach (var e in call.EventsOverview)
+            {
+                Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order.");
+            }
+
             receivedEvents.AddRange(call.EventsOverview);
             foreach (var e in call.EventsOverview)
             {

Tests/FrameworkTests/FrameworkTests.csproj

@@ -18,6 +18,7 @@
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
   </ItemGroup>
 
 </Project>

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -19,11 +19,20 @@ namespace FrameworkTests.NethereumWorkflow
         {
             var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30);
             blocks = new Dictionary<ulong, Block>();
-            
+
+            Block? prev = null;
             for (ulong i = 0; i < 30; i++)
             {
                 ulong d = 100 + i;
-                blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2)));
+                var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2));
+                blocks.Add(d, newBlock);
+
+                if (prev != null)
+                {
+                    prev.Next = newBlock;
+                    newBlock.Previous = prev;
+                }
+                prev = newBlock;
             }
         }
 
@@ -99,23 +108,23 @@ namespace FrameworkTests.NethereumWorkflow
         }
 
         [Test]
-        public void FailsToFindBlockBeforeFrontOfChain()
+        public void FindsGenesisBlockAtFrontOfChain()
         {
             var first = blocks.First().Value;
 
-            var notFound = finder.GetHighestBlockNumberBefore(first.Time);
+            var firstNumber = finder.GetHighestBlockNumberBefore(first.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(firstNumber, Is.EqualTo(first.Number));
         }
 
         [Test]
-        public void FailsToFindBlockAfterTailOfChain()
+        public void FindsCurrentBlockAtTailOfChain()
         {
             var last = blocks.Last().Value;
 
-            var notFound = finder.GetLowestBlockNumberAfter(last.Time);
+            var lastNumber = finder.GetLowestBlockNumberAfter(last.Time);
 
-            Assert.That(notFound, Is.Null);
+            Assert.That(lastNumber, Is.EqualTo(last.Number));
         }
 
         [Test]
@@ -143,13 +152,27 @@ namespace FrameworkTests.NethereumWorkflow
         {
             foreach (var pair in blocks)
             {
-                finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
-                finder.GetHighestBlockNumberBefore(pair.Value.Time);
-                finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
+                var block = pair.Value;
 
-                finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
-                finder.GetLowestBlockNumberAfter(pair.Value.Time);
-                finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
+                AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time));
+                AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter));
+
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore));
+                AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time));
+                AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter));
+            }
+        }
+
+        private void AssertLink(Block? expected, ulong? actual)
+        {
+            if (expected == null)
+            {
+                Assert.That(actual, Is.Null);
+            }
+            else
+            {
+                Assert.That(expected.Number, Is.EqualTo(actual!.Value));
             }
         }
     }
@@ -167,6 +190,9 @@ namespace FrameworkTests.NethereumWorkflow
         public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
         public DateTime JustAfter { get { return Time.AddSeconds(1); } }
 
+        public Block? Next { get; set; }
+        public Block? Previous { get; set; }
+
         public override string ToString()
         {
             return $"[{Number}]";

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -1,135 +0,0 @@
-using Logging;
-using NUnit.Framework;
-using OverwatchTranscript;
-
-namespace FrameworkTests.OverwatchTranscript
-{
-    [TestFixture]
-    public class TranscriptTests
-    {
-        private const string TranscriptFilename = "testtranscript.owts";
-        private const string HeaderKey = "testHeader";
-        private const string HeaderData = "abcdef";
-        private const string EventData0 = "12345";
-        private const string EventData1 = "678";
-        private const string EventData2 = "90";
-        private const string EventData3 = "-=";
-        private readonly DateTime t0 = DateTime.UtcNow;
-        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
-        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
-
-        [Test]
-        public void WriteAndRun()
-        {
-            WriteTranscript();
-            ReadTranscript();
-
-            File.Delete(TranscriptFilename);
-        }
-
-        private void WriteTranscript()
-        {
-            var log = new ConsoleLog();
-            var writer = Transcript.NewWriter(log);
-
-            writer.AddHeader(HeaderKey, new TestHeader
-            {
-                HeaderData = HeaderData
-            });
-
-            writer.Add(t0, new MyEvent
-            {
-                EventData = EventData0
-            });
-            writer.Add(t2, new MyEvent
-            {
-                EventData = EventData3
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData1
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData2
-            });
-
-            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
-
-            writer.Write(TranscriptFilename);
-        }
-
-        private void ReadTranscript()
-        {
-            var reader = Transcript.NewReader(TranscriptFilename);
-
-            var header = reader.GetHeader<TestHeader>(HeaderKey);
-            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
-            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
-            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
-            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
-            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
-
-            var moments = new List<ActivateMoment>();
-            var events = new List<ActivateEvent<MyEvent>>();
-            reader.AddMomentHandler(moments.Add);
-            reader.AddEventHandler<MyEvent>(events.Add);
-
-
-            Assert.That(moments.Count, Is.EqualTo(0));
-            Assert.That(events.Count, Is.EqualTo(0));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(1));
-            Assert.That(events.Count, Is.EqualTo(1));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(2));
-            Assert.That(events.Count, Is.EqualTo(3));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
-            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
-            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
-
-            AssertEvent(events[0], utc: t0, duration: t1 - t0, index: 0, data: EventData0);
-            AssertEvent(events[1], utc: t1, duration: t2 - t1, index: 1, data: EventData1);
-            AssertEvent(events[2], utc: t1, duration: t2 - t1, index: 1, data: EventData2);
-            AssertEvent(events[3], utc: t2, duration: null, index: 2, data: EventData3);
-
-            reader.Close();
-        }
-
-        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
-        {
-            Assert.That(m.Utc, Is.EqualTo(utc));
-            Assert.That(m.Duration, Is.EqualTo(duration));
-            Assert.That(m.Index, Is.EqualTo(index));
-        }
-
-        private void AssertEvent(ActivateEvent<MyEvent> e, DateTime utc, TimeSpan? duration, int index, string data)
-        {
-            Assert.That(e.Moment.Utc, Is.EqualTo(utc));
-            Assert.That(e.Moment.Duration, Is.EqualTo(duration));
-            Assert.That(e.Moment.Index, Is.EqualTo(index));
-            Assert.That(e.Payload.EventData, Is.EqualTo(data));
-        }
-    }
-
-    public class TestHeader
-    {
-        public string HeaderData { get; set; } = string.Empty;
-    }
-
-    public class MyEvent
-    {
-        public string EventData { get; set; } = string.Empty;
-    }
-}

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptLargeTest.cs

@@ -2,7 +2,7 @@
 using NUnit.Framework;
 using OverwatchTranscript;
 
-namespace FrameworkTests.OverwatchTranscript
+namespace FrameworkTests.OverwatchTranscriptTests
 {
     [TestFixture]
     public class TranscriptLargeTests

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptTests.cs

@@ -0,0 +1,229 @@
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using OverwatchTranscript;
+using System.IO.Compression;
+
+namespace FrameworkTests.OverwatchTranscriptTests
+{
+    [TestFixture]
+    public class TranscriptTests
+    {
+        private const string TranscriptFilename = "testtranscript.owts";
+        private const string HeaderKey = "testHeader";
+        private const string HeaderData = "abcdef";
+        private const string EventData0 = "12345";
+        private const string EventData1 = "678";
+        private const string EventData2 = "90";
+        private const string EventData3 = "-=";
+        private readonly DateTime t0 = DateTime.UtcNow;
+        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
+        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
+
+        [Test]
+        [Combinatorial]
+        public void WriteAndRun()
+        {
+            WriteTranscript();
+            AssertFileContent();
+            ReadTranscript();
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private void WriteTranscript()
+        {
+            var log = new ConsoleLog();
+            var writer = Transcript.NewWriter(log);
+
+            writer.AddHeader(HeaderKey, new TestHeader
+            {
+                HeaderData = HeaderData
+            });
+
+            writer.Add(t0, new MyEvent
+            {
+                EventData = EventData0
+            });
+            writer.Add(t2, new MyEvent
+            {
+                EventData = EventData3
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData1
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData2
+            });
+
+            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
+
+            writer.Write(TranscriptFilename);
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var header = reader.GetHeader<TestHeader>(HeaderKey);
+            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
+            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
+
+            var moments = new List<ActivateMoment>();
+            var events = new List<ActivateEvent<MyEvent>>();
+            reader.AddMomentHandler(moments.Add);
+            reader.AddEventHandler<MyEvent>(events.Add);
+
+            var timeout = 10;
+            while (moments.Count < 3 && events.Count < 4)
+            {
+                Thread.Sleep(10);
+                reader.Next();
+
+                timeout--;
+                if (timeout == 0) Assert.Fail("Events not received.");
+            }
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
+            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
+            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
+
+            AssertEvent(events, utc: t0, duration: t1 - t0, index: 0, data: EventData0);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData2);
+            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData1);
+            AssertEvent(events, utc: t2, duration: null, index: 2, data: EventData3);
+
+            reader.Close();
+        }
+
+        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
+        {
+            Assert.That(m.Utc, Is.EqualTo(utc));
+            Assert.That(m.Duration, Is.EqualTo(duration));
+            Assert.That(m.Index, Is.EqualTo(index));
+        }
+
+        private void AssertEvent(List<ActivateEvent<MyEvent>> events, DateTime utc, TimeSpan? duration, int index, string data)
+        {
+            var e = events.SingleOrDefault(e => e.Moment.Utc == utc && e.Payload.EventData == data);
+            if (e == null) Assert.Fail("Event not found");
+
+            Assert.That(e!.Moment.Utc, Is.EqualTo(utc));
+            Assert.That(e!.Moment.Duration, Is.EqualTo(duration));
+            Assert.That(e!.Moment.Index, Is.EqualTo(index));
+            Assert.That(e!.Payload.EventData, Is.EqualTo(data));
+        }
+
+        private void AssertFileContent()
+        {
+            using var zip = ZipFile.OpenRead(TranscriptFilename);
+            Assert.That(zip.Entries.Count, Is.EqualTo(2));
+            foreach (var entry in zip.Entries)
+            {
+                if (entry.Name == "transcript.json")
+                {
+                    var transcript = ZipEntryJson<OverwatchTranscript.OverwatchTranscript>(entry);
+                    AssertTranscript(transcript);
+                }
+                else
+                {
+                    var moments = ZipEntryToMoments(entry);
+                    AssertMoments(moments);
+                }
+            }
+        }
+
+        private void AssertTranscript(OverwatchTranscript.OverwatchTranscript transcript)
+        {
+            Assert.That(transcript.Header.Common.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.Header.Common.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.Header.Common.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.Header.Common.LatestUtc, Is.EqualTo(t2));
+            Assert.That(transcript.Header.Entries.Length, Is.EqualTo(1));
+            Assert.That(transcript.Header.Entries[0].Key, Is.EqualTo(HeaderKey));
+            Assert.That(transcript.Header.Entries[0].Value, Is.EqualTo("{\"HeaderData\":\"abcdef\"}"));
+
+            Assert.That(transcript.MomentReferences.Length, Is.EqualTo(1));
+            Assert.That(transcript.MomentReferences[0].NumberOfMoments, Is.EqualTo(3));
+            Assert.That(transcript.MomentReferences[0].NumberOfEvents, Is.EqualTo(4));
+            Assert.That(transcript.MomentReferences[0].EarliestUtc, Is.EqualTo(t0));
+            Assert.That(transcript.MomentReferences[0].LatestUtc, Is.EqualTo(t2));
+        }
+
+        private void AssertMoments(OverwatchMoment[] moments)
+        {
+            Assert.That(moments.Length, Is.EqualTo(3));
+
+            Assert.That(moments[0].Utc, Is.EqualTo(t0));
+            Assert.That(moments[0].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[0].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[0].Events[0].Payload, Is.EqualTo("{\"EventData\":\"12345\"}"));
+
+            Assert.That(moments[1].Utc, Is.EqualTo(t1));
+            Assert.That(moments[1].Events.Length, Is.EqualTo(2));
+            Assert.That(moments[1].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[1].Events[1].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+
+            // output order is not guaranteed:
+            var payloads = moments[1].Events.Select(e => e.Payload).ToArray();
+            CollectionAssert.AreEquivalent(new[]
+            {
+                "{\"EventData\":\"90\"}",
+                "{\"EventData\":\"678\"}"
+            }, payloads);
+
+            Assert.That(moments[2].Utc, Is.EqualTo(t2));
+            Assert.That(moments[2].Events.Length, Is.EqualTo(1));
+            Assert.That(moments[2].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
+            Assert.That(moments[2].Events[0].Payload, Is.EqualTo("{\"EventData\":\"-=\"}"));
+        }
+
+        private T ZipEntryJson<T>(ZipArchiveEntry? entry)
+        {
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+            var json = reader.ReadToEnd();
+            var result = JsonConvert.DeserializeObject<T>(json);
+            if (result == null) Assert.Fail("didn't deserialize");
+            return result!;
+        }
+
+        private OverwatchMoment[] ZipEntryToMoments(ZipArchiveEntry? entry)
+        {
+            var result = new List<OverwatchMoment>();
+            if (entry == null) Assert.Fail("entry is null");
+            using var stream = entry!.Open();
+            using var reader = new StreamReader(stream);
+
+            var line = reader.ReadLine();
+            while (!string.IsNullOrEmpty(line))
+            {
+                var moment = JsonConvert.DeserializeObject<OverwatchMoment>(line);
+                if (moment != null) result.Add(moment);
+                line = reader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+    }
+
+    public class TestHeader
+    {
+        public string HeaderData { get; set; } = string.Empty;
+    }
+
+    public class MyEvent
+    {
+        public string EventData { get; set; } = string.Empty;
+    }
+}

Tests/FrameworkTests/Utils/EmojiMapsTests.cs

@@ -0,0 +1,28 @@
+using NUnit.Framework;
+using System.Text;
+using TestNetRewarder;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EmojiMapsTests
+    {
+        private readonly Random random = new Random();
+        private readonly EmojiMaps maps = new EmojiMaps();
+
+        [Test]
+        public void GeneratesConsistentStrings(
+            [Values(1, 5, 10, 20)] int inputLength,
+            [Values(1, 2, 3, 5)] int outLength)
+        {
+            var buffer = new byte[inputLength];
+            random.NextBytes(buffer);
+            var input = Encoding.ASCII.GetString(buffer);
+
+            var out1 = maps.StringToEmojis(input, outLength);
+            var out2 = maps.StringToEmojis(input, outLength);
+            
+            Assert.That(out1, Is.EqualTo(out2));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncoding.cs

@@ -0,0 +1,334 @@
+namespace FrameworkTests.Utils
+{
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public RunUpdate ExpandToInclude(int index)
+        {
+            if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}");
+            if (index == (Start + Length))
+            {
+                Length++;
+                return new RunUpdate();
+            }
+            if (index == (Start - 1))
+            {
+                return new RunUpdate(
+                    newRuns: [new Run(Start - 1, Length + 1)],
+                    removeRuns: [this]
+                );
+            }
+            throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}");
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(
+                        newRuns: Array.Empty<Run>(),
+                        removeRuns: [this]
+                    );
+                }
+                return new RunUpdate(
+                    newRuns: [new Run(Start + 1, Length - 1)],
+                    removeRuns: [this]
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(
+                newRuns: [new Run(index + 1, newRunLength)],
+                removeRuns: Array.Empty<Run>()
+            );
+        }
+
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+
+        public override string ToString()
+        {
+            return $"[{Start},{Length}]";
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is Run run &&
+                   Start == run.Start &&
+                   Length == run.Length;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Start, Length);
+        }
+
+        public static bool operator ==(Run? obj1, Run? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2);
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
+
+    public partial class IndexSet
+    {
+        private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+        public IndexSet()
+        {
+        }
+
+        public IndexSet(int[] indices)
+        {
+            foreach (var i in indices) Set(i);
+        }
+
+        public static IndexSet FromRunLengthEncoded(int[] rle)
+        {
+            var set = new IndexSet();
+            for (var i = 0; i < rle.Length; i += 2)
+            {
+                var start = rle[i];
+                var length = rle[i + 1];
+                set.runs.Add(start, new Run(start, length));
+            }
+
+            return set;
+        }
+
+        public bool IsSet(int index)
+        {
+            if (runs.ContainsKey(index)) return true;
+
+            var run = GetRunAt(index);
+            if (run == null) return false;
+            return true;
+        }
+
+        public void Set(int index)
+        {
+            if (IsSet(index)) return;
+
+            var runBefore = GetRunAt(index - 1);
+            var runAfter = GetRunExact(index + 1);
+
+            if (runBefore == null)
+            {
+                if (runAfter == null)
+                {
+                    CreateNewRun(index);
+                }
+                else
+                {
+                    HandleUpdate(runAfter.ExpandToInclude(index));
+                }
+            }
+            else
+            {
+                if (runAfter == null)
+                {
+                    HandleUpdate(runBefore.ExpandToInclude(index));
+                }
+                else
+                {
+                    // new index will connect runBefore with runAfter. We merge!
+                    HandleUpdate(new RunUpdate(
+                        newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)],
+                        removeRuns: [runBefore, runAfter]
+                    ));
+                }
+            }
+        }
+
+        public void Unset(int index)
+        {
+            if (runs.ContainsKey(index))
+            {
+                HandleUpdate(runs[index].Unset(index));
+            }
+            else
+            {
+                var run = GetRunAt(index);
+                if (run == null) return;
+                HandleUpdate(run.Unset(index));
+            }
+        }
+
+        public void Iterate(Action<int> onIndex)
+        {
+            foreach (var run in runs.Values)
+            {
+                run.Iterate(onIndex);
+            }
+        }
+
+        public int[] RunLengthEncoded()
+        {
+            return Encode().ToArray();
+        }
+
+        public override string ToString()
+        {
+            return string.Join("&", runs.Select(r => r.ToString()).ToArray());
+        }
+
+        private IEnumerable<int> Encode()
+        {
+            foreach (var pair in runs)
+            {
+                yield return pair.Value.Start;
+                yield return pair.Value.Length;
+            }
+        }
+
+        private Run? GetRunAt(int index)
+        {
+            foreach (var run in runs.Values)
+            {
+                if (run.Includes(index)) return run;
+            }
+            return null;
+        }
+
+        private Run? GetRunExact(int index)
+        {
+            if (runs.ContainsKey(index)) return runs[index];
+            return null;
+        }
+
+        private void HandleUpdate(RunUpdate runUpdate)
+        {
+            foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+        }
+
+        private void CreateNewRun(int index)
+        {
+            if (runs.ContainsKey(index + 1))
+            {
+                var length = runs[index + 1].Length + 1;
+                runs.Add(index, new Run(index, length));
+                runs.Remove(index + 1);
+            }
+            else
+            {
+                runs.Add(index, new Run(index, 1));
+            }
+        }
+    }
+
+    public partial class IndexSet
+    {
+        public IndexSet Overlap(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public IndexSet Merge(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(result.Set);
+            other.Iterate(result.Set);
+            return result;
+        }
+
+        public IndexSet Without(IndexSet other)
+        {
+            var result = new IndexSet();
+            Iterate(i =>
+            {
+                if (!other.IsSet(i)) result.Set(i);
+            });
+            return result;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (obj is IndexSet set)
+            {
+                if (set.runs.Count != runs.Count) return false;
+                foreach (var pair in runs)
+                {
+                    if (!set.runs.ContainsKey(pair.Key)) return false;
+                    if (set.runs[pair.Key] != pair.Value) return false;
+                }
+                return true;
+            }
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(runs);
+        }
+
+        public static bool operator ==(IndexSet? obj1, IndexSet? obj2)
+        {
+            if (ReferenceEquals(obj1, obj2)) return true;
+            if (ReferenceEquals(obj1, null)) return false;
+            if (ReferenceEquals(obj2, null)) return false;
+            return obj1.Equals(obj2);
+        }
+        public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2);
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs

@@ -0,0 +1,94 @@
+using NUnit.Framework;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class RunLengthEncodingLogicalTests
+    {
+        [Test]
+        public void EqualityTest()
+        {
+            var setA = new IndexSet([1, 2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.EqualTo(setB));
+            Assert.That(setA == setB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var setA = new IndexSet([1, 2, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var setA = new IndexSet([1, 2, 3]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest3()
+        {
+            var setA = new IndexSet([2, 3, 4, 5]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void InequalityTest()
+        {
+            var setA = new IndexSet([2, 3, 4]);
+            var setB = new IndexSet([1, 2, 3, 4]);
+
+            Assert.That(setA, Is.Not.EqualTo(setB));
+            Assert.That(setA != setB);
+        }
+
+        [Test]
+        public void Overlap()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([3, 4, 5, 11]);
+
+            var set = setA.Overlap(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Merge()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]);
+
+            var set = setA.Merge(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+
+        [Test]
+        public void Without()
+        {
+            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
+            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
+            var expectedSet = new IndexSet([1, 2, 14]);
+
+            var set = setA.Without(setB);
+
+            Assert.That(set, Is.EqualTo(expectedSet));
+        }
+    }
+}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -1,12 +1,40 @@
 using NUnit.Framework;
-using NUnit.Framework.Interfaces;
-using static FrameworkTests.Utils.RunLengthEncodingTests;
 
 namespace FrameworkTests.Utils
 {
     [TestFixture]
     public class RunLengthEncodingRunTests
     {
+        [Test]
+        public void EqualityTest()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 4);
+
+            Assert.That(runA, Is.EqualTo(runB));
+            Assert.That(runA == runB);
+        }
+
+        [Test]
+        public void InequalityTest1()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(1, 5);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
+        [Test]
+        public void InequalityTest2()
+        {
+            var runA = new Run(1, 4);
+            var runB = new Run(2, 4);
+
+            Assert.That(runA, Is.Not.EqualTo(runB));
+            Assert.That(runA != runB);
+        }
+
         [Test]
         [Combinatorial]
         public void RunIncludes(
@@ -33,23 +61,58 @@ namespace FrameworkTests.Utils
         }
 
         [Test]
-        public void RunExpandToInclude()
+        public void RunExpandThrowsWhenIndexNotAdjacent()
         {
             var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
             Assert.That(run.Includes(2));
             Assert.That(run.Includes(4));
             Assert.That(!run.Includes(5));
 
-            Assert.That(run.ExpandToInclude(1), Is.False);
-            Assert.That(run.ExpandToInclude(2), Is.False);
-            Assert.That(run.ExpandToInclude(4), Is.False);
-            Assert.That(run.ExpandToInclude(6), Is.False);
+            Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf<Exception>());
+        }
 
-            Assert.That(run.ExpandToInclude(5), Is.True);
+        [Test]
+        public void RunExpandThrowsWhenIndexAlreadyIncluded()
+        {
+            var run = new Run(2, 3);
+            Assert.That(!run.Includes(1));
+            Assert.That(run.Includes(2));
+            Assert.That(run.Includes(4));
+            Assert.That(!run.Includes(5));
+
+            Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf<Exception>());
+            Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf<Exception>());
+        }
+
+        [Test]
+        public void RunExpandToIncludeAfter()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(5);
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
             Assert.That(run.Includes(5));
             Assert.That(!run.Includes(6));
         }
 
+        [Test]
+        public void RunExpandToIncludeBefore()
+        {
+            var run = new Run(2, 3);
+            var update = run.ExpandToInclude(1);
+
+            Assert.That(update, Is.Not.Null);
+            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
+            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
+
+            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
+            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
+            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
+        }
+
         [Test]
         public void RunCanUnsetLastIndex()
         {
@@ -100,94 +163,9 @@ namespace FrameworkTests.Utils
         {
             var run = new Run(2, 4);
             var seen = new List<int>();
-            run.Iterate(i => seen.Add(i));
+            run.Iterate(seen.Add);
 
             CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
         }
     }
-
-    public class Run
-    {
-        public Run(int start, int length)
-        {
-            Start = start;
-            Length = length;
-        }
-
-        public int Start { get; }
-        public int Length { get; private set; }
-
-        public bool Includes(int index)
-        {
-            return index >= Start && index < (Start + Length);
-        }
-
-        public bool ExpandToInclude(int index)
-        {
-            if (index == (Start + Length))
-            {
-                Length++;
-                return true;
-            }
-            return false;
-        }
-
-        public RunUpdate Unset(int index)
-        {
-            if (!Includes(index))
-            {
-                return new RunUpdate();
-            }
-
-            if (index == Start)
-            {
-                // First index: Replace self with new run at next index, unless empty.
-                if (Length == 1)
-                {
-                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
-                }
-                return new RunUpdate(
-                    newRuns: new[] { new Run(Start + 1, Length - 1) },
-                    removeRuns: new[] { this }
-                );
-            }
-
-            if (index == (Start + Length - 1))
-            {
-                // Last index: Become one smaller.
-                Length--;
-                return new RunUpdate();
-            }
-
-            // Split:
-            var newRunLength = (Start + Length - 1) - index;
-            Length = index - Start;
-            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
-        }
-        
-        public void Iterate(Action<int> action)
-        {
-            for (var i = 0; i < Length; i++)
-            {
-                action(Start + i);
-            }
-        }
-    }
-
-    public class RunUpdate
-    {
-        public RunUpdate()
-            : this(Array.Empty<Run>(), Array.Empty<Run>())
-        {
-        }
-
-        public RunUpdate(Run[] newRuns, Run[] removeRuns)
-        {
-            NewRuns = newRuns;
-            RemoveRuns = removeRuns;
-        }
-
-        public Run[] NewRuns { get; }
-        public Run[] RemoveRuns { get; }
-    }
 }

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -1,9 +1,4 @@
-using Logging;
-using Microsoft.VisualStudio.TestPlatform.Common;
-using NuGet.Frameworks;
-using NUnit.Framework;
-using System.Collections.Concurrent;
-using System.Numerics;
+using NUnit.Framework;
 using Utils;
 
 namespace FrameworkTests.Utils
@@ -119,6 +114,19 @@ namespace FrameworkTests.Utils
             }, encoded);
         }
 
+        [Test]
+        public void SetIndexBetweenRuns()
+        {
+            var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 });
+            set.Set(11);
+            var encoded = set.RunLengthEncoded();
+
+            CollectionAssert.AreEqual(new[]
+            {
+                8, 7
+            }, encoded);
+        }
+
         [Test]
         public void SetIndexAfterRun()
         {
@@ -201,120 +209,5 @@ namespace FrameworkTests.Utils
             all.Sort();
             return all.ToArray();
         }
-
-        public class IndexSet
-        {
-            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
-
-            public IndexSet()
-            {
-            }
-
-            public IndexSet(int[] indices)
-            {
-                foreach (var i in indices) Set(i);
-            }
-
-            public static IndexSet FromRunLengthEncoded(int[] rle)
-            {
-                var set = new IndexSet();
-                for (var i = 0; i < rle.Length; i += 2)
-                {
-                    var start = rle[i];
-                    var length = rle[i + 1];
-                    set.runs.Add(start, new Run(start, length));
-                }
-
-                return set;
-            }
-
-            public bool IsSet(int index)
-            {
-                if (runs.ContainsKey(index)) return true;
-
-                var run = GetRunBefore(index);
-                if (run == null) return false;
-
-                return run.Includes(index);
-            }
-
-            public void Set(int index)
-            {
-                if (runs.ContainsKey(index)) return;
-
-                var run = GetRunBefore(index);
-                if (run == null || !run.ExpandToInclude(index))
-                {
-                    CreateNewRun(index);
-                }
-            }
-
-            public void Unset(int index)
-            {
-                if (runs.ContainsKey(index))
-                {
-                    HandleUpdate(runs[index].Unset(index));
-                }
-                else
-                {
-                    var run = GetRunBefore(index);
-                    if (run == null) return;
-                    HandleUpdate(run.Unset(index));
-                }
-            }
-
-            public void Iterate(Action<int> onIndex)
-            {
-                foreach (var run in runs.Values)
-                {
-                    run.Iterate(onIndex);
-                }
-            }
-
-            public int[] RunLengthEncoded()
-            {
-                return Encode().ToArray();
-            }
-
-            private IEnumerable<int> Encode()
-            {
-                foreach (var pair in runs)
-                {
-                    yield return pair.Value.Start;
-                    yield return pair.Value.Length;
-                }
-            }
-
-            private Run? GetRunBefore(int index)
-            {
-                Run? result = null;
-                foreach (var pair in runs)
-                {
-                    if (pair.Key < index) result = pair.Value;
-                    else return result;
-                }
-                return result;
-            }
-
-            private void HandleUpdate(RunUpdate runUpdate)
-            {
-                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
-                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
-            }
-
-            private void CreateNewRun(int index)
-            {
-                if (runs.ContainsKey(index + 1))
-                {
-                    var length = runs[index + 1].Length + 1;
-                    runs.Add(index, new Run(index, length));
-                    runs.Remove(index + 1);
-                }
-                else
-                {
-                    runs.Add(index, new Run(index, 1));
-                }
-            }
-        }
     }
 }

Tools/AutoClient/Purchaser.cs

@@ -68,7 +68,7 @@ namespace AutoClient
                 var filename = Guid.NewGuid().ToString().ToLowerInvariant();
                 {
                     using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkAsync(cid);
+                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
                     fileResponse.Stream.CopyTo(fileStream);
                 }
                 var time = sw.Elapsed;
@@ -84,8 +84,15 @@ namespace AutoClient
         private async Task<string> StartNewPurchase()
         {
             var file = await CreateFile();
-            var cid = await UploadFile(file);
-            return await RequestStorage(cid);
+            try
+            {
+                var cid = await UploadFile(file);
+                return await RequestStorage(cid);
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
         }
 
         private async Task<string> CreateFile()
@@ -93,6 +100,18 @@ namespace AutoClient
             return await app.Generator.Generate();
         }
 
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
         private async Task<ContentId> UploadFile(string filename)
         {
             using var fileStream = File.OpenRead(filename);

Tools/AutoClient/README.md

@@ -6,7 +6,7 @@ Can generate random images or random data of a specified size.
 
 ## How to run
 
-- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- dotnet 8.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
 - docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
 
 ## Configuration options

Tools/AutoClient/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/BiblioTech/BaseCommand.cs

@@ -1,7 +1,6 @@
 using Discord.WebSocket;
 using BiblioTech.Options;
 using Discord;
-using k8s.KubeConfigModels;
 
 namespace BiblioTech
 {

Tools/BiblioTech/BiblioTech.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CodexCidChecker.cs

@@ -0,0 +1,198 @@
+using CodexOpenApi;
+using IdentityModel.Client;
+using Utils;
+
+namespace BiblioTech
+{
+    public class CodexCidChecker
+    {
+        private static readonly string nl = Environment.NewLine;
+        private readonly Configuration config;
+        private CodexApi? currentCodexNode;
+
+        public CodexCidChecker(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public async Task<CheckResponse> PerformCheck(string cid)
+        {
+            if (string.IsNullOrEmpty(config.CodexEndpoint))
+            {
+                return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
+            }
+
+            try
+            {
+                var codex = GetCodex();
+                var nodeCheck = await CheckCodex(codex);
+                if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
+
+                return await PerformCheck(codex, cid);
+            }
+            catch (Exception ex)
+            {
+                return new CheckResponse(false, "Internal server error", ex.ToString());
+            }
+        }
+
+        private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
+        {
+            try
+            {
+                var manifest = await codex.DownloadNetworkManifestAsync(cid);
+                return SuccessMessage(manifest);
+            }
+            catch (ApiException apiEx)
+            {
+                if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
+                if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
+                return UnexpectedReturnCode(apiEx.Response);
+            }
+            catch (Exception ex)
+            {
+                return UnexpectedException(ex);
+            }
+        }
+
+        #region Response formatting
+
+        private CheckResponse SuccessMessage(DataItem content)
+        {
+            return FormatResponse(
+                success: true,
+                title: $"Success: '{content.Cid}'",
+                error: "",
+                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"blockSize: {content.Manifest.BlockSize} bytes",
+                $"protected: {content.Manifest.Protected}"
+            );
+        }
+
+        private CheckResponse UnexpectedException(Exception ex)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Unexpected error",
+                error: ex.ToString(),
+                content: "Details will be sent to the bot-admin channel."
+            );
+        }
+
+        private CheckResponse UnexpectedReturnCode(string response)
+        {
+            var msg = "Unexpected return code. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Unexpected return code",
+                error: msg,
+                content: msg
+            );
+        }
+
+        private CheckResponse FailedToFetch(string response)
+        {
+            var msg = "Failed to download content. Response: " + response;
+            return FormatResponse(
+                success: false,
+                title: "Could not download content",
+                error: msg,
+                msg,
+                $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
+            );
+        }
+
+        private CheckResponse CidFormatInvalid(string response)
+        {
+            return FormatResponse(
+                success: false,
+                title: "Invalid format",
+                error: "",
+                content: "Provided CID is not formatted correctly."
+            );
+        }
+
+        private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
+        {
+            var msg = string.Join(nl,
+                new string[]
+                {
+                    title,
+                    "```"
+                }
+                .Concat(content)
+                .Concat(new string[]
+                {
+                    "```"
+                })
+            ) + nl + nl;
+
+            return new CheckResponse(success, msg, error);
+        }
+
+        #endregion
+
+        #region Codex Node API 
+
+        private CodexApi GetCodex()
+        {
+            if (currentCodexNode == null) currentCodexNode = CreateCodex();
+            return currentCodexNode;
+        }
+
+        private async Task<bool> CheckCodex(CodexApi codex)
+        {
+            try
+            {
+                var info = await currentCodexNode!.GetDebugInfoAsync();
+                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
+                return true;
+            }
+            catch (Exception e)
+            {
+                return false;
+            }
+        }
+
+        private CodexApi CreateCodex()
+        {
+            var endpoint = config.CodexEndpoint;
+            var splitIndex = endpoint.LastIndexOf(':');
+            var host = endpoint.Substring(0, splitIndex);
+            var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
+
+            var address = new Address(
+                host: host,
+                port: port
+            );
+
+            var client = new HttpClient();
+            if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
+            {
+                var tokens = config.CodexEndpointAuth.Split(':');
+                if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
+                client.SetBasicAuthentication(tokens[0], tokens[1]);    
+            }
+
+            var codex = new CodexApi(client);
+            codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return codex;
+        }
+
+        #endregion
+    }
+
+    public class CheckResponse
+    {
+        public CheckResponse(bool success, string message, string error)
+        {
+            Success = success;
+            Message = message;
+            Error = error;
+        }
+
+        public bool Success { get; }
+        public string Message { get; }
+        public string Error { get; }
+    }
+}

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -0,0 +1,38 @@
+using BiblioTech.Options;
+
+namespace BiblioTech.Commands
+{
+    public class CheckCidCommand : BaseCommand
+    {
+        private readonly StringOption cidOption = new StringOption(
+            name: "cid",
+            description: "Codex Content-Identifier",
+            isRequired: true);
+        private readonly CodexCidChecker checker;
+
+        public CheckCidCommand(CodexCidChecker checker)
+        {
+            this.checker = checker;
+        }
+
+        public override string Name => "check";
+        public override string StartingMessage => RandomBusyMessage.Get();
+        public override string Description => "Checks if content is available in the testnet.";
+        public override CommandOption[] Options => new[] { cidOption };
+
+        protected override async Task Invoke(CommandContext context)
+        {
+            var user = context.Command.User;
+            var cid = await cidOption.Parse(context);
+            if (string.IsNullOrEmpty(cid))
+            {
+                await context.Followup("Option 'cid' was not received.");
+                return;
+            }
+
+            var response = await checker.PerformCheck(cid);
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
+            await context.Followup(response.Message);
+        }
+    }
+}

Tools/BiblioTech/Configuration.cs

@@ -38,6 +38,12 @@ namespace BiblioTech
         [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
         public int NoDiscord { get; set; } = 0;
 
+        [Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")]
+        public string CodexEndpoint { get; set; } = "http://localhost:8080";
+
+        [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
+        public string CodexEndpointAuth { get; set; } = "";
+
         public string EndpointsPath => Path.Combine(DataPath, "endpoints");
         public string UserDataPath => Path.Combine(DataPath, "users");
         public string LogPath => Path.Combine(DataPath, "logs");

Tools/BiblioTech/Program.cs

@@ -3,7 +3,6 @@ using BiblioTech.Commands;
 using BiblioTech.Rewards;
 using Discord;
 using Discord.WebSocket;
-using DiscordRewards;
 using Logging;
 
 namespace BiblioTech
@@ -81,6 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
+            var checker = new CodexCidChecker(Config);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();
@@ -90,6 +90,7 @@ namespace BiblioTech
                 sprCommand,
                 associateCommand,
                 notifyCommand,
+                new CheckCidCommand(checker),
                 new AdminCommand(sprCommand, replacement)
             );
 

Tools/BiblioTech/Rewards/ChainEventsSender.cs

@@ -1,4 +1,5 @@
 using Discord.WebSocket;
+using DiscordRewards;
 using Logging;
 
 namespace BiblioTech.Rewards
@@ -16,24 +17,17 @@ namespace BiblioTech.Rewards
             this.eventsChannel = eventsChannel;
         }
 
-        public async Task ProcessChainEvents(string[] eventsOverview)
+        public async Task ProcessChainEvents(ChainEventMessage[] eventsOverview, string[] errors)
         {
+            await SendErrorsToAdminChannel(errors);
+
             if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
             try
             {
                 await Task.Run(async () =>
                 {
                     var users = Program.UserRepo.GetAllUserData();
-
-                    foreach (var e in eventsOverview)
-                    {
-                        if (!string.IsNullOrEmpty(e))
-                        {
-                            var @event = ApplyReplacements(users, e);
-                            await eventsChannel.SendMessageAsync(@event);
-                            await Task.Delay(1000);
-                        }
-                    }
+                    await SendChainEventsInOrder(eventsOverview, eventsChannel, users);
                 });
             }
             catch (Exception ex)
@@ -42,6 +36,37 @@ namespace BiblioTech.Rewards
             }
         }
 
+        private async Task SendErrorsToAdminChannel(string[] errors)
+        {
+            try
+            {
+                foreach (var error in errors)
+                {
+                    await Program.AdminChecker.SendInAdminChannel(error);
+                }
+            }
+            catch (Exception exc)
+            {
+                log.Error("Failed to send error messages to admin channel. " + exc);
+                Environment.Exit(1);
+            }
+        }
+
+        private async Task SendChainEventsInOrder(ChainEventMessage[] eventsOverview, SocketTextChannel eventsChannel, UserData[] users)
+        {
+            eventsOverview = eventsOverview.OrderBy(e => e.BlockNumber).ToArray();
+            foreach (var e in eventsOverview)
+            {
+                var msg = e.Message;
+                if (!string.IsNullOrEmpty(msg))
+                {
+                    var @event = ApplyReplacements(users, msg);
+                    await eventsChannel.SendMessageAsync(@event);
+                    await Task.Delay(300);
+                }
+            }
+        }
+
         private string ApplyReplacements(UserData[] users, string msg)
         {
             var result = ApplyUserAddressReplacements(users, msg);

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -31,7 +31,7 @@ namespace BiblioTech.Rewards
                 await ProcessRewards(rewards);
             }
 
-            await eventsSender.ProcessChainEvents(rewards.EventsOverview);
+            await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors);
         }
 
         private async Task ProcessRewards(GiveRewardsCommand rewards)

Tools/BiblioTech/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/CsvCombiner/CsvCombiner.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Tools/CsvCombiner/Program.cs

@@ -0,0 +1,71 @@
+using Logging;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"];
+        var p = new Program(args[0]);
+        p.Run();
+    }
+
+    private static readonly ILog log = new ConsoleLog();
+    private string path;
+
+    private readonly Dictionary<string, List<string>> combine = new Dictionary<string, List<string>>();
+
+    public Program(string path)
+    {
+        this.path = path;
+    }
+
+    private void Run()
+    {
+        Log("Starting in " + path);
+
+        var files = Directory.GetFiles(path)
+            .Where(f => f.ToLowerInvariant().EndsWith(".csv")).ToArray();
+
+        foreach (var file in files)
+        {
+            AddToMap(file);
+        }
+
+        var i = 0;
+        foreach (var pair in combine)
+        {
+            var list = pair.Value;
+            list.Insert(0, pair.Key);
+
+            File.WriteAllLines(Path.Combine(path, "combine_" + i + ".csv"), list.ToArray());
+            i++;
+        }
+
+        Log("done");
+    }
+
+    private void AddToMap(string file)
+    {
+        var lines = File.ReadAllLines(file);
+        if (lines.Length > 1)
+        {
+            var header = lines[0];
+            var list = GetList(header);
+            list.AddRange(lines.Skip(1));
+        }
+    }
+
+    private List<string> GetList(string header)
+    {
+        if (!combine.ContainsKey(header))
+        {
+            combine.Add(header, new List<string>());
+        }
+        return combine[header];
+    }
+
+    private void Log(string msg)
+    {
+        log.Log(msg);
+    }
+}

Tools/KeyMaker/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/MarketInsights/AverageHistory.cs

@@ -1,6 +1,5 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
-using Nethereum.Model;
 using TestNetRewarder;
 using Utils;
 
@@ -40,7 +39,7 @@ namespace MarketInsights
 
         private MarketTimeSegment BuildContribution(TimeRange timeRange)
         {
-            var builder = new ContributionBuilder(timeRange);
+            var builder = new ContributionBuilder(appState.Log, timeRange);
             mux.Handlers.Add(builder);
             chainState.Update(timeRange.To);
             mux.Handlers.Remove(builder);

Tools/MarketInsights/ContributionBuilder.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin.ChainMonitor;
 using GethPlugin;
+using Logging;
 using System.Numerics;
 using Utils;
 
@@ -8,14 +9,16 @@ namespace MarketInsights
     public class ContributionBuilder : IChainStateChangeHandler
     {
         private readonly MarketTimeSegment segment = new MarketTimeSegment();
+        private readonly ILog log;
 
-        public ContributionBuilder(TimeRange timeRange)
+        public ContributionBuilder(ILog log, TimeRange timeRange)
         {
             segment = new MarketTimeSegment
             {
                 FromUtc = timeRange.From,
                 ToUtc = timeRange.To
             };
+            this.log = log;
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
@@ -51,6 +54,15 @@ namespace MarketInsights
         {
         }
 
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+            log.Error(msg);
+        }
+
         public MarketTimeSegment GetSegment()
         {
             return segment;

Tools/TestNetRewarder/BotClient.cs

@@ -18,7 +18,6 @@ namespace TestNetRewarder
         public async Task<bool> IsOnline()
         {
             var result = await HttpGet();
-            log.Log("Is DiscordBot online: " + result);
             return result == "Pong";
         }
 

Tools/TestNetRewarder/EmojiMaps.cs

@@ -0,0 +1,83 @@
+using Utils;
+
+namespace TestNetRewarder
+{
+    public class EmojiMaps
+    {
+        private readonly string[] emojis = new[]
+        {
+            // yellow
+            "😀",
+            "🌻",
+            "🍋",
+            "🧀",
+            "🌔",
+            "⭐",
+            "⚡",
+            "🏆",
+            // red
+            "💘",
+            "🦞",
+            "🌹",
+            "🍒",
+            "🫖", // teapot
+            "⛩",
+            "🚗",
+            "🔥",
+            // green
+            "🐊",
+            "🦎",
+            "🐛",
+            "🌳",
+            "🍀",
+            "🧩",
+            "🔋",
+            "♻",
+            // blue
+            "💙",
+            "🐳",
+            "🐟",
+            "🍉",
+            "🧊",
+            "🌐",
+            "⚓",
+            "🌀",
+        };
+
+        public string NewRequest => "🐟";
+        public string Started => "🦈";
+        public string SlotFilled => "🟢";
+        public string SlotFreed => "⭕";
+        public string SlotReservationsFull => "☑️";
+        public string Finished => "✅";
+        public string Cancelled => "🚫";
+        public string Failed => "❌";
+
+        public string StringToEmojis(string input, int outLength)
+        {
+            if (outLength < 1) outLength = 1;
+
+            var result = "";
+            var segmentLength = input.Length / outLength;
+            if (segmentLength < 1)
+            {
+                return StringToEmojis(input + input, outLength);
+            }
+            for (var i = 0; i < outLength; i++)
+            {
+                var segment = input.Substring(i * segmentLength, segmentLength);
+                result += SelectOne(segment);
+            }
+
+            return result;
+        }
+
+        private string SelectOne(string segment)
+        {
+            var index = 0;
+            foreach (var c in segment) index += Convert.ToInt32(c);
+            index = index % emojis.Length;
+            return emojis[index];
+        }
+    }
+}

Tools/TestNetRewarder/EventsFormatter.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
+using DiscordRewards;
 using GethPlugin;
 using System.Globalization;
 using System.Numerics;
@@ -10,24 +11,28 @@ namespace TestNetRewarder
     public class EventsFormatter : IChainStateChangeHandler
     {
         private static readonly string nl = Environment.NewLine;
-        private readonly List<string> events = new List<string>();
+        private readonly List<ChainEventMessage> events = new List<ChainEventMessage>();
+        private readonly List<string> errors = new List<string>();
+        private readonly EmojiMaps emojiMaps = new EmojiMaps();
 
-        public string[] GetEvents()
+        public ChainEventMessage[] GetEvents()
         {
             var result = events.ToArray();
             events.Clear();
             return result;
         }
 
-        public void AddError(string error)
+        public string[] GetErrors()
         {
-            AddBlock("📢 **Error**", error);
+            var result = errors.ToArray();
+            errors.Clear();
+            return result;
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
         {
             var request = requestEvent.Request;
-            AddRequestBlock(requestEvent, "New Request",
+            AddRequestBlock(requestEvent, $"{emojiMaps.NewRequest} New Request",
                 $"Client: {request.Client}",
                 $"Content: {request.Request.Content.Cid}",
                 $"Duration: {BigIntToDuration(request.Request.Ask.Duration)}",
@@ -42,27 +47,27 @@ namespace TestNetRewarder
 
         public void OnRequestCancelled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Cancelled");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Cancelled} Cancelled");
         }
 
         public void OnRequestFailed(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Failed");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Failed} Failed");
         }
 
         public void OnRequestFinished(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Finished");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Finished} Finished");
         }
 
         public void OnRequestFulfilled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, "Started");
+            AddRequestBlock(requestEvent, $"{emojiMaps.Started} Started");
         }
 
         public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, "Slot Filled",
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFilled} Slot Filled",
                 $"Host: {host}",
                 $"Slot Index: {slotIndex}"
             );
@@ -70,24 +75,46 @@ namespace TestNetRewarder
 
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, "Slot Freed",
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFreed} Slot Freed",
                 $"Slot Index: {slotIndex}"
             );
         }
 
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            AddRequestBlock(requestEvent, $"{emojiMaps.SlotReservationsFull} Slot Reservations Full",
+                $"Slot Index: {slotIndex}"
+            );
+        }
+        
+        public void OnError(string msg)
+        {
+            errors.Add(msg);
+        }
+
         private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
         {
             var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
-            var title = $"{blockNumber} **{eventName}** `{requestEvent.Request.Request.Id}`";
-            AddBlock(title, content);
+            var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}";
+            AddBlock(requestEvent.Block.BlockNumber, title, content);
         }
 
-        private void AddBlock(string title, params string[] content)
+        private void AddBlock(ulong blockNumber, string title, params string[] content)
         {
-            events.Add(FormatBlock(title, content));
+            events.Add(FormatBlock(blockNumber, title, content));
         }
 
-        private string FormatBlock(string title, params string[] content)
+        private ChainEventMessage FormatBlock(ulong blockNumber, string title, params string[] content)
+        {
+            var msg = FormatBlockMessage(title, content);
+            return new ChainEventMessage
+            {
+                BlockNumber = blockNumber,
+                Message = msg
+            };
+        }
+
+        private string FormatBlockMessage(string title, string[] content)
         {
             if (content == null || !content.Any())
             {
@@ -113,6 +140,13 @@ namespace TestNetRewarder
             return utc.ToString("yyyy-MM-dd HH:mm:ss UTC", CultureInfo.InvariantCulture);
         }
 
+        private string FormatRequestId(RequestEvent requestEvent)
+        {
+            return
+                $"({emojiMaps.StringToEmojis(requestEvent.Request.Request.Id, 3)})" +
+                $"`{requestEvent.Request.Request.Id}`";
+        }
+
         private string BigIntToDuration(BigInteger big)
         {
             var span = TimeSpan.FromSeconds((int)big);

Tools/TestNetRewarder/Processor.cs

@@ -48,7 +48,7 @@ namespace TestNetRewarder
             {
                 var msg = "Exception processing time segment: " + ex;
                 log.Error(msg); 
-                eventsFormatter.AddError(msg);
+                eventsFormatter.OnError(msg);
                 throw;
             }
         }
@@ -58,8 +58,9 @@ namespace TestNetRewarder
             var numberOfChainEvents = chainState.Update(timeRange.To);
 
             var events = eventsFormatter.GetEvents();
+            var errors = eventsFormatter.GetErrors();
 
-            var request = builder.Build(events);
+            var request = builder.Build(events, errors);
             if (request.HasAny())
             {
                 await client.SendRewards(request);

Tools/TestNetRewarder/RequestBuilder.cs

@@ -19,7 +19,7 @@ namespace TestNetRewarder
             }
         }
 
-        public GiveRewardsCommand Build(string[] lines)
+        public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors)
         {
             var result = new GiveRewardsCommand
             {
@@ -28,7 +28,8 @@ namespace TestNetRewarder
                     RewardId = p.Key,
                     UserAddresses = p.Value.Select(v => v.Address).ToArray()
                 }).ToArray(),
-                EventsOverview = lines
+                EventsOverview = lines,
+                Errors = errors
             };
 
             rewards.Clear();

Tools/TestNetRewarder/RewardCheck.cs

@@ -72,6 +72,14 @@ namespace TestNetRewarder
         {
         }
 
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
+
         private void GiveReward(RewardConfig reward, EthAddress receiver)
         {
             giver.Give(reward, receiver);

Tools/TestNetRewarder/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/TranscriptAnalysis/CsvWriter.cs

@@ -0,0 +1,136 @@
+using Logging;
+
+namespace TranscriptAnalysis
+{
+    public class CsvWriter
+    {
+        private readonly ILog log;
+
+        public CsvWriter(ILog log)
+        {
+            this.log = log;
+        }
+
+        public ICsv CreateNew()
+        {
+            return new Csv();
+        }
+
+        public void Write(ICsv csv, string filename)
+        {
+            var c = (Csv)csv;
+
+            using var file = File.OpenWrite(filename);
+            using var writer = new StreamWriter(file);
+            c.CreateLines(writer.WriteLine);
+
+            log.Log($"CSV written to: '{filename}'");
+        }
+    }
+
+    public interface ICsv
+    {
+        ICsvColumn GetColumn(string title, float defaultValue);
+        ICsvColumn GetColumn(string title, string defaultValue);
+        void AddRow(params CsvCell[] cells);
+    }
+
+    public class Csv : ICsv
+    {
+        private readonly string Sep = ",";
+        private readonly List<CsvColumn> columns = new List<CsvColumn>();
+        private readonly List<CsvRow> rows = new List<CsvRow>();
+
+        public ICsvColumn GetColumn(string title, float defaultValue)
+        {
+            return GetColumn(title, defaultValue.ToString());
+        }
+
+        public ICsvColumn GetColumn(string title, string defaultValue)
+        {
+            var column = columns.SingleOrDefault(c => c.Title == title);
+            if (column == null)
+            {
+                column = new CsvColumn(title, defaultValue);
+                columns.Add(column);
+            }
+            return column;
+        }
+
+        public void AddRow(params CsvCell[] cells)
+        {
+            rows.Add(new CsvRow(cells));
+        }
+
+        public void CreateLines(Action<string> onLine)
+        {
+            CreateHeaderLine(onLine);
+            foreach (var row in rows)
+            {
+                CreateRowLine(row, onLine);
+            }
+        }
+
+        private void CreateHeaderLine(Action<string> onLine)
+        {
+            onLine(string.Join(Sep, columns.Select(c => c.Title).ToArray()));
+        }
+
+        private void CreateRowLine(CsvRow row, Action<string> onLine)
+        {
+            onLine(string.Join(Sep, columns.Select(c => GetRowCellValue(row, c)).ToArray()));
+        }
+
+        private string GetRowCellValue(CsvRow row, CsvColumn column)
+        {
+            var cell = row.Cells.SingleOrDefault(c => c.Column == column);
+            if (cell == null) return column.DefaultValue;
+            return cell.Value;
+        }
+    }
+
+    public class CsvCell
+    {
+        public CsvCell(ICsvColumn column, float value)
+            : this(column, value.ToString())
+        {
+        }
+
+        public CsvCell(ICsvColumn column, string value)
+        {
+            Column = column;
+            Value = value;
+        }
+
+        public ICsvColumn Column { get; }
+        public string Value { get; }
+    }
+
+    public interface ICsvColumn
+    {
+        string Title { get; }
+        string DefaultValue { get; }
+    }
+
+    public class CsvColumn : ICsvColumn
+    {
+        public CsvColumn(string title, string defaultValue)
+        {
+            Title = title;
+            DefaultValue = defaultValue;
+        }
+
+        public string Title { get; }
+        public string DefaultValue { get; }
+    }
+
+    public class CsvRow
+    {
+        public CsvRow(CsvCell[] cells)
+        {
+            Cells = cells;
+        }
+
+        public CsvCell[] Cells { get; }
+    }
+}

Tools/TranscriptAnalysis/Program.cs

@@ -9,8 +9,6 @@ public static class Program
 
     public static void Main(string[] args)
     {
-        //args = new[] { "D:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net7.0\\CodexTestLogs\\2024-08\\06\\08-24-45Z_ThreeClientTest\\SwarmTest_SwarmTest.owts" };
-
         Log("Transcript Analysis");
         if (!args.Any())
         {
@@ -33,7 +31,7 @@ public static class Program
         };
 
         var header = reader.GetHeader<OverwatchCodexHeader>("cdx_h");
-        var receivers = new ReceiverSet(log, reader, header);
+        var receivers = new ReceiverSet(args[0], log, reader, header);
         receivers.InitAll();
 
         var processor = new Processor(log, reader);

Tools/TranscriptAnalysis/ReceiverSet.cs

@@ -7,7 +7,7 @@ namespace TranscriptAnalysis
 {
     public interface IEventReceiver
     {
-        void Init(ILog log, OverwatchCodexHeader header);
+        void Init(string sourceFilename, ILog log, OverwatchCodexHeader header);
         void Finish();
     }
 
@@ -18,13 +18,15 @@ namespace TranscriptAnalysis
 
     public class ReceiverSet
     {
+        private readonly string sourceFilename;
         private readonly ILog log;
         private readonly ITranscriptReader reader;
         private readonly OverwatchCodexHeader header;
         private readonly List<IEventReceiver> receivers = new List<IEventReceiver>();
 
-        public ReceiverSet(ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
+        public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
         {
+            this.sourceFilename = sourceFilename;
             this.log = log;
             this.reader = reader;
             this.header = header;
@@ -53,7 +55,7 @@ namespace TranscriptAnalysis
             mux.Add(receiver);
 
             receivers.Add(receiver);
-            receiver.Init(log, header);
+            receiver.Init(sourceFilename, log, header);
         }
 
         // We use a mux here because, for each time we call reader.AddEventHandler,

Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs

@@ -8,29 +8,38 @@ namespace TranscriptAnalysis.Receivers
     {
         protected ILog log { get; private set; } = new NullLog();
         protected OverwatchCodexHeader Header { get; private set; } = null!;
+        protected CsvWriter CsvWriter { get; private set; }
+        protected string SourceFilename { get; private set; } = string.Empty;
 
         public abstract string Name { get; }
         public abstract void Receive(ActivateEvent<T> @event);
         public abstract void Finish();
 
-        public void Init(ILog log, OverwatchCodexHeader header)
+        protected BaseReceiver()
+        {
+            CsvWriter = new CsvWriter(log);
+        }
+
+        public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header)
         {
             this.log = new LogPrefixer(log, $"({Name}) ");
             Header = header;
+            SourceFilename = sourceFilename;
         }
 
-        protected string GetPeerId(int nodeIndex)
+        protected string? GetPeerId(int nodeIndex)
         {
-            return GetIdentity(nodeIndex).PeerId;
+            return GetIdentity(nodeIndex)?.PeerId;
         }
 
-        protected string GetName(int nodeIndex)
+        protected string? GetName(int nodeIndex)
         {
-            return GetIdentity(nodeIndex).Name;
+            return GetIdentity(nodeIndex)?.Name;
         }
 
-        protected CodexNodeIdentity GetIdentity(int nodeIndex)
+        protected CodexNodeIdentity? GetIdentity(int nodeIndex)
         {
+            if (nodeIndex < 0) return null;
             return Header.Nodes[nodeIndex];
         }
 

Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs

@@ -5,6 +5,9 @@ namespace TranscriptAnalysis.Receivers
 {
     public class DuplicateBlocksReceived : BaseReceiver<OverwatchCodexEvent>
     {
+        public static List<int> Counts = new List<int>();
+        private long uploadSize;
+
         public override string Name => "BlocksReceived";
 
         public override void Receive(ActivateEvent<OverwatchCodexEvent> @event)
@@ -13,11 +16,17 @@ namespace TranscriptAnalysis.Receivers
             {
                 Handle(@event.Payload, @event.Payload.BlockReceived);
             }
+            if (@event.Payload.FileUploaded != null)
+            {
+                var uploadEvent = @event.Payload.FileUploaded;
+                uploadSize = uploadEvent.ByteSize;
+            }
         }
 
         public override void Finish()
         {
             Log("Number of BlockReceived events seen: " + seen);
+            var csv = CsvWriter.CreateNew();
 
             var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value));
             var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value));
@@ -31,13 +40,26 @@ namespace TranscriptAnalysis.Receivers
                 }
             }
 
+            if (Counts.Any()) throw new Exception("Should be empty");
+
             float t = totalReceived;
+            csv.GetColumn("numNodes", Header.Nodes.Length);
+            csv.GetColumn("filesize", uploadSize.ToString());
+            var receiveCountColumn = csv.GetColumn("receiveCount", 0.0f);
+            var occuranceColumn = csv.GetColumn("occurance", 0.0f);
             occurances.PrintContinous((i, count) =>
             {
                 float n = count;
                 float p = 100.0f * (n / t);
                 Log($"Block received {i} times = {count}x ({p}%)");
+                Counts.Add(count);
+                csv.AddRow(
+                    new CsvCell(receiveCountColumn, i),
+                    new CsvCell(occuranceColumn, count)
+                );
             });
+
+            CsvWriter.Write(csv, SourceFilename + "_blockduplicates.csv");
         }
 
         private int seen = 0;
@@ -46,6 +68,7 @@ namespace TranscriptAnalysis.Receivers
         private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived)
         {
             var receiverPeerId = GetPeerId(payload.NodeIdentity);
+            if (receiverPeerId == null) return;
             var blockAddress = blockReceived.BlockAddress;
             seen++;
 

Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs

@@ -14,6 +14,8 @@ namespace TranscriptAnalysis.Receivers
         {
             var peerId = GetPeerId(@event.Payload.NodeIdentity);
             var name = GetName(@event.Payload.NodeIdentity);
+            if (peerId == null) return;
+            if (name == null) return;
 
             if (!seen.Contains(peerId))
             {

Tools/TranscriptAnalysis/Receivers/NodesDegree.cs

@@ -46,6 +46,7 @@ namespace TranscriptAnalysis.Receivers
 
         private readonly Dictionary<string, Node> dialingNodes = new Dictionary<string, Node>();
         private readonly Dictionary<string, Dial> dials = new Dictionary<string, Dial>();
+        private long uploadSize;
 
         public override string Name => "NodesDegree";
 
@@ -54,12 +55,20 @@ namespace TranscriptAnalysis.Receivers
             if (@event.Payload.DialSuccessful != null)
             {
                 var peerId = GetPeerId(@event.Payload.NodeIdentity);
+                if (peerId == null) return;
                 AddDial(peerId, @event.Payload.DialSuccessful.TargetPeerId);
             }
+            if (@event.Payload.FileUploaded != null)
+            {
+                var uploadEvent = @event.Payload.FileUploaded;
+                uploadSize = uploadEvent.ByteSize;
+            }
         }
 
         public override void Finish()
         {
+            var csv = CsvWriter.CreateNew();
+
             var numNodes = dialingNodes.Count;
             var redialOccurances = new OccuranceMap();
             foreach (var dial in dials.Values)
@@ -80,12 +89,22 @@ namespace TranscriptAnalysis.Receivers
             });
 
             float tot = numNodes;
+            csv.GetColumn("numNodes", Header.Nodes.Length);
+            csv.GetColumn("filesize", uploadSize.ToString());
+            var degreeColumn = csv.GetColumn("degree", 0.0f);
+            var occuranceColumn = csv.GetColumn("occurance", 0.0f);
             degreeOccurances.Print((i, count) =>
             {
                 float n = count;
                 float p = 100.0f * (n / tot);
                 Log($"Degree: {i} = {count}x ({p}%)");
+                csv.AddRow(
+                    new CsvCell(degreeColumn, i),
+                    new CsvCell(occuranceColumn, n)
+                );
             });
+
+            CsvWriter.Write(csv, SourceFilename + "_nodeDegrees.csv");
         }
 
         private void AddDial(string peerId, string targetPeerId)

cs-codex-dist-testing.sln

@@ -76,6 +76,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -202,6 +204,10 @@ Global
 		{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -237,6 +243,7 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
+		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:7.0
+FROM mcr.microsoft.com/dotnet/sdk:8.0
 
 COPY --chmod=0755 docker/docker-entrypoint.sh /
 

docker/deployandrun.Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:7.0
+FROM mcr.microsoft.com/dotnet/sdk:8.0
 
 RUN apt-get update && apt-get install -y screen
 WORKDIR /app

docs/Automation.md

@@ -104,7 +104,7 @@ GitHub --> CI --> Kubernetes -->  Job                  |
     spec:
       containers:
       - name: dotnet
-        image: mcr.microsoft.com/dotnet/sdk:7.0
+        image: mcr.microsoft.com/dotnet/sdk:8.0
         env:
         - name: RUNNERLOCATION
           value: InternalToCluster

docs/Continuous-Tests.md

@@ -44,7 +44,7 @@
     spec:
       containers:
       - name: runner
-        image: mcr.microsoft.com/dotnet/sdk:7.0
+        image: mcr.microsoft.com/dotnet/sdk:8.0
         env:
         - name: KUBECONFIG
           value: /opt/kubeconfig.yaml

docs/LOCALSETUP.md

@@ -4,7 +4,7 @@
 These steps will help you set up everything you need to run and debug the tests on your local system.
 
 ### Installing the requirements.
-1. Install dotnet v7.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net7.0` with your version.)
+1. Install dotnet v8.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net8.0` with your version.)
 1. Set up a nice C# IDE or plugin for your current IDE.
 1. Install docker desktop.
 1. In the docker-desktop settings, enable kubernetes. (This might take a few minutes.)