Merge branch 'master' into findbug/networkpeer-crash

.gitattributes

@@ -1,10 +0,0 @@
-# Set default behavior to automatically normalize line endings.
-* text=auto
-
-# Force bash scripts to always use lf line endings so that if a repo is accessed
-# in Unix via a file share from Windows, the scripts will work.
-*.sh text eol=lf
-
-# Likewise, force cmd and batch scripts to always use crlf
-*.cmd text eol=crlf
-*.bat text eol=crlf

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,8 +3,7 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
-        public string[] Errors { get; set; } = Array.Empty<string>();
+        public string[] EventsOverview { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -17,10 +16,4 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
-
-    public class ChainEventMessage
-    {
-        public ulong BlockNumber { get; set; }
-        public string Message { get; set; } = string.Empty;
-    }
 }

Framework/KubernetesWorkflow/Configuration.cs

@@ -21,5 +21,8 @@ namespace KubernetesWorkflow
 
         [JsonIgnore]
         public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
+
+        [JsonIgnore]
+        public Func<string?, string?> Replacer { get; set; } = s => s;
     }
 }

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,11 +11,13 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
+        private readonly Func<string?, string?> replacer;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace,
+            Func<string?, string?> replacer)
         {
             this.log = log;
             this.config = config;
@@ -23,6 +25,7 @@ namespace KubernetesWorkflow
             this.podName = podName;
             this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
+            this.replacer = replacer;
             cts = new CancellationTokenSource();
         }
 
@@ -92,7 +95,7 @@ namespace KubernetesWorkflow
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
             var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
-            handler.Log(stream);
+            handler.Log(stream, replacer);
         }
     }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -12,10 +12,11 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly K8sCluster cluster;
         private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly Func<string?, string?> replacer;
         private readonly K8sClient client;
         public const string PodLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace, Func<string?, string?> replacer)
         {
             this.log = log;
             this.cluster = cluster;
@@ -23,6 +24,7 @@ namespace KubernetesWorkflow
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sNamespace = k8sNamespace;
+            this.replacer = replacer;
         }
 
         public void Dispose()
@@ -64,7 +66,7 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous, Func<string?, string?> replacer)
         {
             log.Debug();
 
@@ -72,7 +74,7 @@ namespace KubernetesWorkflow
             var recipeName = container.Recipe.Name;
 
             using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
-            logHandler.Log(stream);
+            logHandler.Log(stream, replacer);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -906,7 +908,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true, replacer);
 
                 throw new Exception(msg);
             }
@@ -952,7 +954,7 @@ namespace KubernetesWorkflow
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace, replacer);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,23 +1,22 @@
 using Logging;
-using Utils;
 
 namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
-        void Log(Stream log);
+        void Log(Stream log, Func<string?, string?> replacer);
     }
 
     public abstract class LogHandler : ILogHandler
     {
-        public void Log(Stream log)
+        public void Log(Stream log, Func<string?, string?> replacer)
         {
             using var reader = new StreamReader(log);
             var line = reader.ReadLine();
             while (line != null)
             {
-                ProcessLine(line);
-                line = reader.ReadLine();
+                line = replacer(reader.ReadLine());
+                if (line != null) ProcessLine(line);
             }
         }
 
@@ -41,11 +40,6 @@ namespace KubernetesWorkflow
 
         protected override void ProcessLine(string line)
         {
-            foreach (var replacement in BaseLog.replacements)
-            {
-                line = replacement.Apply(line);
-            }
-
             LogFile.WriteRaw(line);
         }
     }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -28,16 +28,17 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly string k8sNamespace;
+        private readonly Func<string?, string?> replacer;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace, Func<string?, string?> replacer)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.k8sNamespace = k8sNamespace;
-
+            this.replacer = replacer;
             locationProvider = new LocationProvider(log, K8s);
         }
 
@@ -119,7 +120,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
             });
         }
 
@@ -131,7 +132,7 @@ namespace KubernetesWorkflow
 
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
             });
 
             return new DownloadedLog(logHandler, container.Name);
@@ -257,7 +258,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
                 action(controller);
                 controller.Dispose();
             }
@@ -272,7 +273,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
                 var result = action(controller);
                 controller.Dispose();
                 return result;

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride), configuration.Replacer);
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/Logging/BaseLog.cs

@@ -16,7 +16,7 @@ namespace Logging
         public static bool EnableDebugLogging { get; set; } = false;
 
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        public static List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
+        private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private LogFile? logFile;
 
         public BaseLog()

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -29,8 +29,7 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment < bounds.Genesis.Utc) return null;
-            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
+            if (moment <= bounds.Genesis.Utc) return null;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -39,8 +38,7 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment > bounds.Current.Utc) return null;
-            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
+            if (moment >= bounds.Current.Utc) return null;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,7 +7,6 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
-        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -29,8 +28,7 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
-        private bool sourceIsEmpty;
+        private bool stopping;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -44,38 +42,34 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
-        public void Update()
-        {
-            if (IsEmpty) return;
-            while (topQueue.Count == 0)
-            {
-                UpdateIsEmpty();
-                if (IsEmpty) return;
-
-                itemDequeued.Set();
-                itemEnqueued.WaitOne(200);
-            }
-        }
-
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            if (topQueue.TryPeek(out BucketTop? top))
+            while (true)
             {
-                return top.Utc;
+                UpdateIsEmpty();
+                if (IsEmpty) return null;
+                if (topQueue.TryPeek(out BucketTop? top))
+                {
+                    return top.Utc;
+                }
             }
-            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-            if (topQueue.TryDequeue(out BucketTop? top))
+
+            while (true)
             {
-                itemDequeued.Set();
-                return top;
+                UpdateIsEmpty();
+                if (IsEmpty) return null;
+                if (topQueue.TryDequeue(out BucketTop? top))
+                {
+                    itemDequeued.Set();
+                    return top;
+                }
             }
-            return null;
         }
 
         private void ReadBucket()
@@ -91,25 +85,23 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
-                        itemEnqueued.Set();
                     }
                     else
                     {
-                        sourceIsEmpty = true;
-                        UpdateIsEmpty();
+                        stopping = true;
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne(5000);
+                itemDequeued.WaitOne();
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
-            if (!IsEmpty && allEmpty)
+            var empty = stopping && topQueue.IsEmpty;
+            if (!IsEmpty && empty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,8 +24,6 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
-                foreach (var b in buckets) b.Update();
-
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/Utils/PluginPathUtils.cs

@@ -1,41 +0,0 @@
-namespace Utils
-{
-    public static class PluginPathUtils
-    {
-        private const string ProjectPluginsFolderName = "ProjectPlugins";
-        private static string projectPluginsDir = string.Empty;
-
-        public static string ProjectPluginsDir
-        {
-            get
-            {
-                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
-                return projectPluginsDir;
-            }
-        }
-
-        private static string FindProjectPluginsDir()
-        {
-            var current = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var localFolders = Directory.GetDirectories(current);
-                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-                if (projectPluginsFolders.Length == 1)
-                {
-                    return projectPluginsFolders.Single();
-                }
-
-                var parent = Directory.GetParent(current);
-                if (parent == null)
-                {
-                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                    Console.WriteLine(msg);
-                    throw new Exception(msg);
-                }
-
-                current = parent.FullName;
-            }
-        }
-    }
-}

Framework/Utils/RandomUtils.cs

@@ -5,16 +5,6 @@
         private static readonly Random random = new Random();
         private static readonly object @lock = new object();
 
-        public static T GetOneRandom<T>(this T[] items)
-        {
-            lock (@lock)
-            {
-                var i = random.Next(0, items.Length);
-                var result = items[i];
-                return result;
-            }
-        }
-
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
             lock (@lock)

Framework/Utils/Retry.cs

@@ -71,10 +71,6 @@
                         task();
                         return;
                     }
-                    catch (OperationCanceledException)
-                    {
-                        return;
-                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,5 +1,4 @@
 using CodexContractsPlugin.Marketplace;
-using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -13,8 +12,7 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed,
-            SlotReservationsFullEventDTO[] slotReservationsFull
+            SlotFreedEventDTO[] slotFreed
             )
         {
             BlockInterval = blockInterval;
@@ -24,9 +22,6 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
-            SlotReservationsFull = slotReservationsFull;
-
-            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -36,8 +31,21 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
-        public IHasBlock[] All { get; }
+
+        public IHasBlock[] All
+        {
+            get
+            {
+                var all = new List<IHasBlock>();
+                all.AddRange(Requests);
+                all.AddRange(Fulfilled);
+                all.AddRange(Cancelled);
+                all.AddRange(Failed);
+                all.AddRange(SlotFilled);
+                all.AddRange(SlotFreed);
+                return all.ToArray();
+            }
+        }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -58,19 +66,8 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents(),
-                events.GetSlotReservationsFull()
+                events.GetSlotFreedEvents()
             );
         }
-
-        private T[] ConcatAll<T>(params T[][] arrays)
-        {
-            var result = Array.Empty<T>();
-            foreach (var array in arrays)
-            {
-                result = result.Concat(array).ToArray();
-            }
-            return result;
-        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -16,9 +16,6 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
-        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
-
-        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -69,11 +66,7 @@ namespace CodexContractsPlugin.ChainMonitor
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-            {
-                var msg = "Attempt to update ChainState with set of events from before its current record.";
-                handler.OnError(msg);
-                throw new Exception(msg);
-            }
+                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
 
             log.Log($"ChainState updating: {events.BlockInterval}");
 
@@ -116,7 +109,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -124,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -132,7 +125,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -140,7 +133,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -149,21 +142,13 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event);
+            var r = FindRequest(@event.RequestId);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
-        private void ApplyEvent(SlotReservationsFullEventDTO @event)
-        {
-            var r = FindRequest(@event);
-            if (r == null) return;
-            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
-            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
-        }
-
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -177,23 +162,10 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(IHasRequestId request)
+        private ChainStateRequest? FindRequest(byte[] requestId)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
-            if (r == null)
-            {
-                var blockNumber = "unknown";
-                if (request is IHasBlock blk)
-                {
-                    blockNumber = blk.Block.BlockNumber.ToString();
-                }
-
-                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
-                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
-
-                log.Error(msg);
-                handler.OnError(msg);
-            }
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
+            if (r == null) log.Log("Unable to find request by ID!");
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,5 +1,10 @@
 using GethPlugin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -46,15 +51,5 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
-
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
-        }
-
-        public void OnError(string msg)
-        {
-            foreach (var handler in Handlers) handler.OnError(msg);
-        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,13 +32,5 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
-
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-        }
-
-        public void OnError(string msg)
-        {
-        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,7 +1,6 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
 using NethereumWorkflow.BlockUtils;
 using Utils;
@@ -17,7 +16,6 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
-        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -40,32 +38,49 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events.Select(e =>
+            return events
+            .Select(e =>
             {
-                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                var result = requestEvent.ReturnValue1;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                result.RequestId = e.Event.RequestId;
-                return result;
-            }).ToArray();
+                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(SetBlockOnEvent).ToArray();
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(SetBlockOnEvent).ToArray();
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(SetBlockOnEvent).ToArray();
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -83,20 +98,12 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(SetBlockOnEvent).ToArray();
-        }
-
-        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
-        {
-            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(SetBlockOnEvent).ToArray();
-        }
-
-        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
-        {
-            var result = e.Event;
-            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-            return result;
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                return result;
+            }).ToArray();
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -10,12 +10,7 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public interface IHasRequestId
-    {
-        byte[] RequestId { get; set; }
-    }
-
-    public partial class Request : RequestBase, IHasBlock, IHasRequestId
+    public partial class Request : RequestBase, IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -33,38 +28,32 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
+    public partial class RequestFulfilledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
+    public partial class RequestCancelledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
+    public partial class RequestFailedEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
+    public partial class SlotFilledEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
-    {
-        [JsonIgnore]
-        public BlockTimeEntry Block { get; set; }
-    }
-
-    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
+    public partial class SlotFreedEventDTO : IHasBlock
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,6 +1,4 @@
-using Utils;
-
-namespace CodexContractsPlugin
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -43,10 +41,24 @@ namespace CodexContractsPlugin
 
         private string GetMarketplaceFilePath()
         {
-            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
-            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
-            return path;
+            var here = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var path = GetMarketplaceFile(here);
+                if (path != null) return path;
+
+                var parent = Directory.GetParent(here);
+                var up = parent?.FullName;
+                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
+                here = up;
+            }
+        }
+
+        private string? GetMarketplaceFile(string root)
+        {
+            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (File.Exists(path)) return path;
+            return null;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-fb25372";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,14 +3,13 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
-using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "39-0C-32-A3-EA-90-4F-29-1C-67-12-F1-D5-BE-31-67-8D-90-43-1E-F2-02-63-5B-0C-49-F7-1E-E5-EC-F7-00";
+        private const string OpenApiYamlHash = "6B-94-24-A4-D5-01-6F-12-E9-34-74-36-80-57-7A-3A-79-8C-E8-02-68-B7-05-DA-50-A0-5C-B1-02-B9-AE-C6";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -22,9 +21,8 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
-            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -73,23 +71,10 @@ namespace CodexPlugin
                 return;
             }
 
-            OverwriteOpenApiYaml(containerApi);
-
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
-        private void OverwriteOpenApiYaml(string containerApi)
-        {
-            Log("API compatibility check failed. Updating CodexPlugin...");
-            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
-            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
-
-            File.Delete(openApiFilePath);
-            File.WriteAllText(openApiFilePath, containerApi);
-            Log("CodexPlugin/openapi.yaml has been updated.");
-        }
-
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -73,7 +73,7 @@ namespace CodexPlugin
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkStreamAsync(contentId),
+                api => api.DownloadNetworkAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@@ -82,31 +82,31 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync("", "")));
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex(api => api.OfferStorageAsync(body));
+            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
         public StorageAvailability[] GetAvailabilities()
         {
-            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
             return mapper.Map(collection);
         }
 
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex(api => api.SpaceAsync());
+            var space = OnCodex<Space>(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,20 +7,8 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage =
-        //"codexstorage/nim-codex:0.1.7-dist-tests"; // => 20/20: 17 seconds 10/10: 3 seconds
-        //"codexstorage/nim-codex:sha-2a25460-dist-tests"; // PR => 20/20: 17 seconds
-        //"thatbenbierens/nim-codex:blockexcpr1"; // PR with revert of "Fixes issue where only wants of type block are stored in peerContext" => 20/20: 17 seconds
-        //"thatbenbierens/nim-codex:blockexprecreate"; // v0.1.7 with patch => 20/20: 19 seconds
-        //"thatbenbierens/nim-codex:blockexprecreate016"; // v0.1.6 with patch => 20/20: 19 seconds 10/10: 2 seconds
-        //"thatbenbierens/nim-codex:blockexchprtinker7";
-        //"thatbenbierens/nim-codex:blkexc9"; // wow-fast
-
-        "thatbenbierens/nim-codex:asyncprofile5break"; // asynced trees.
-
-        //blocks are stored, blocks are resolved
-        //store-stream does not continue. node too busy???
-
+        private const string DefaultDockerImage = "thatbenbierens/nim-codex:netpeerdebug6";
+            //"codexstorage/nim-codex:0.1.4";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -264,27 +264,10 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
-            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                // Type of stream generated by openAPI client does not support timeouts.
-                var start = DateTime.UtcNow;
-                var cts = new CancellationTokenSource();
-                var downloadTask = Task.Run(() =>
-                {
-                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                    downloadStream.CopyTo(fileStream);
-                }, cts.Token);
-                
-                while (DateTime.UtcNow - start < timeout)
-                {
-                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
-                    if (downloadTask.IsCompletedSuccessfully) return;
-                    Thread.Sleep(100);
-                }
-
-                cts.Cancel();
-                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
+                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                downloadStream.CopyTo(fileStream);
             }
             catch
             {

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,8 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = Map(debugInfo.Codex),
-                Table = Map(debugInfo.Table)
+                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
+                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
             };
         }
 
@@ -136,45 +136,47 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion Map(CodexVersion obj)
+        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
         {
             return new DebugInfoVersion
             {
-                Version = obj.Version,
-                Revision = obj.Revision
+                Version = StringOrEmpty(obj, "version"),
+                Revision = StringOrEmpty(obj, "revision")
             };
         }
 
-        private DebugInfoTable Map(PeersTable obj)
+        private DebugInfoTable MapDebugInfoTable(JObject obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = Map(obj.LocalNode),
-                Nodes = Map(obj.Nodes)
+                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
+                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
             };
         }
 
-        private DebugInfoTableNode Map(Node? token)
+        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
         {
-            if (token == null) return new DebugInfoTableNode();
+            var obj = token as JObject;
+            if (obj == null) return new DebugInfoTableNode();
+
             return new DebugInfoTableNode
             {
-                Address = token.Address,
-                NodeId = token.NodeId,
-                PeerId = token.PeerId,
-                Record = token.Record,
-                Seen = token.Seen
+                Address = StringOrEmpty(obj, "address"),
+                NodeId = StringOrEmpty(obj, "nodeId"),
+                PeerId = StringOrEmpty(obj, "peerId"),
+                Record = StringOrEmpty(obj, "record"),
+                Seen = Bool(obj, "seen")
             };
         }
 
-        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
+        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(Map).ToArray();
+            return nodes.Select(MapDebugInfoTableNode).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -83,46 +83,33 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
+    ErasureParameters:
+      type: object
+      properties:
+        totalChunks:
+          type: integer
+
+    PoRParameters:
+      description: Parameters for Proof of Retrievability
+      type: object
+      properties:
+        u:
+          type: string
+        publicKey:
+          type: string
+        name:
+          type: string
+
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-
-    Node:
-      type: object
-      properties:
-        nodeId:
-          type: string
-        peerId:
-          type: string
-        record:
-          type: string
-        address:
-          type: string
-        seen:
-          type: boolean
-
-    CodexVersion:
-      type: object
-      properties:
-        version:
-          type: string
-          example: v0.1.7
-        revision:
-          type: string
-          example: 0c647d8
-
-    PeersTable:
-      type: object
-      properties:
-        localNode:
-          $ref: "#/components/schemas/Node"
-        nodes:
-          type: array
-          items:
-            $ref: "#/components/schemas/Node"
+        erasure:
+          $ref: "#/components/schemas/ErasureParameters"
+        por:
+          $ref: "#/components/schemas/PoRParameters"
 
     DebugInfo:
       type: object
@@ -138,10 +125,6 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
-        table:
-          $ref: "#/components/schemas/PeersTable"
-        codex:
-          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -357,19 +340,6 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
-        filename:
-          type: string
-          description: "The original name of the uploaded content (optional)"
-          example: codex.png
-        mimetype:
-          type: string
-          description: "The original mimetype of the uploaded content (optional)"
-          example: image/png
-        uploadedAt:
-          type: integer
-          format: int64
-          description: "The UTC upload timestamp in seconds"
-          example: 1729244192
 
     Space:
       type: object
@@ -381,15 +351,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
+          description: "Maximum storage space used by the node"
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
+          description: "Amount of storage space currently in use"
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
+          description: "Amount of storage space reserved"
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -443,21 +413,6 @@ paths:
       summary: "Lists manifest CIDs stored locally in node."
       tags: [ Data ]
       operationId: listData
-      parameters:
-        - name: content-type
-          in: header
-          required: false
-          description: The content type of the file. Must be valid.
-          schema:
-            type: string
-            example: "image/png"
-        - name: content-disposition
-          in: header
-          required: false
-          description: The content disposition used to send the filename.
-          schema:
-            type: string
-            example: "attachment; filename=\"codex.png\""
       responses:
         "200":
           description: Retrieved list of content CIDs
@@ -470,8 +425,6 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
-        "422":
-          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
@@ -523,36 +476,10 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
-    post:
-      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
-      tags: [ Data ]
-      operationId: downloadNetwork
-      parameters:
-        - in: path
-          name: cid
-          required: true
-          schema:
-              $ref: "#/components/schemas/Cid"
-          description: "File to be downloaded."
-      responses:
-        "200":
-          description: Manifest information for download that has been started.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/DataItem"
-        "400":
-          description: Invalid CID is specified
-        "404":
-          description: Failed to download dataset manifest
-        "500":
-          description: Well it was bad-bad
-
-  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetworkStream
+      operationId: downloadNetwork
       parameters:
         - in: path
           name: cid
@@ -575,32 +502,6 @@ paths:
         "500":
           description: Well it was bad-bad
 
-  "/data/{cid}/network/manifest":
-    get:
-      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
-      tags: [ Data ]
-      operationId: downloadNetworkManifest
-      parameters:
-        - in: path
-          name: cid
-          required: true
-          schema:
-              $ref: "#/components/schemas/Cid"
-          description: "File for which the manifest is to be downloaded."
-      responses:
-        "200":
-          description: Manifest information.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/DataItem"
-        "400":
-          description: Invalid CID is specified
-        "404":
-          description: Failed to download dataset manifest
-        "500":
-          description: Well it was bad-bad
-
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -846,7 +747,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/spr":
+  "/node/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -864,7 +765,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/peerid":
+  "/node/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -912,4 +813,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -7,8 +7,4 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
-  </ItemGroup>
-
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,6 +1,5 @@
 using System.Security.Cryptography;
 using System.Text;
-using Utils;
 
 public static class Program
 {
@@ -41,9 +40,32 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
-        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
-        return folder;
+        var current = Directory.GetCurrentDirectory();
+
+        while (true)
+        {
+            var localFolders = Directory.GetDirectories(current);
+            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+            if (projectPluginsFolders.Length == 1)
+            {
+                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
+            }
+            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
+            if (codexPluginFolders.Length == 1)
+            {
+                return codexPluginFolders.Single();
+            }
+
+            var parent = Directory.GetParent(current);
+            if (parent == null)
+            {
+                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                Console.WriteLine(msg);
+                throw new Exception(msg);
+            }
+
+            current = parent.FullName;
+        }
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics GetAllMetrics();
+        Metrics? GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics GetAllMetrics()
+        public Metrics? GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,10 +54,11 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet GetMostRecent(string metricName)
+        private MetricsSet? GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            return result.Sets.Last();
+            if (result == null) return null;
+            return result.Sets.LastOrDefault();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
         {
-            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
+            return starter.CollectMetricsFor(scrapeTargets);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
+            if (response == null) return null;
 
             var result = new Metrics
             {
@@ -44,20 +44,19 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics GetMetrics(string metricName)
+        public Metrics? GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
+            if (response == null) return null;
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var instanceString = GetInstanceStringForNode(target);
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
-            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
+            if (response.status != "success") return null;
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -81,30 +80,16 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
-            };
-        }
-
-        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
-        {
-            var result = new MetricsSet
-            {
-                Name = r.metric.__name__,
-                Instance = r.metric.instance,
-                Values = MapMultipleValues(r.values)
-            };
-
-            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
-            {
-                result.AsyncProfiler = new AsyncProfilerMetrics
+                Sets = response.data.result.Select(r =>
                 {
-                    File = r.metric.file,
-                    Line = r.metric.line,
-                    Proc = r.metric.proc
-                };
-            }
-
-            return result;
+                    return new MetricsSet
+                    {
+                        Name = r.metric.__name__,
+                        Instance = r.metric.instance,
+                        Values = MapMultipleValues(r.values)
+                    };
+                }).ToArray()
+            };
         }
 
         private MetricsSetValue[] MapSingleValue(object[] value)
@@ -235,28 +220,14 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
-        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            var prefix = "";
-            if (AsyncProfiler != null)
-            {
-                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
-            }
-
-            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
-    public class AsyncProfilerMetrics
-    {
-        public string File { get; set; } = string.Empty;
-        public string Line { get; set; } = string.Empty;
-        public string Proc { get; set; } = string.Empty;
-    }
-
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -292,10 +263,6 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
-        // Async profiler output.
-        public string? file { get; set; } = null;
-        public string? line { get; set; } = null;
-        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,16 +48,12 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
         {
-            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
-            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
-            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
-
             var config = "";
             config += "global:\n";
-            config += $"  scrape_interval: {secs}s\n";
-            config += $"  scrape_timeout: {secs}s\n";
+            config += "  scrape_interval: 10s\n";
+            config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v8.0  
+Dotnet: v7.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

Tests/CodexTests/BasicTests/AsyncProfiling.cs

@@ -1,85 +0,0 @@
-using NUnit.Framework;
-using MetricsPlugin;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class AsyncProfiling : CodexDistTest
-    {
-        [Test]
-        public void AsyncProfileMetricsPlz()
-        {
-            var node = StartCodex(s => s.EnableMetrics());
-            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single();
-
-            var file = GenerateTestFile(100.MB());
-            node.UploadFile(file);
-
-            Thread.Sleep(10000);
-
-            var profilerMetrics = new AsyncProfileMetrics(metrics.GetAllMetrics());
-
-            var log = GetTestLog();
-            log.Log($"{nameof(profilerMetrics.CallCount)} = {profilerMetrics.CallCount.Highest()}");
-            log.Log($"{nameof(profilerMetrics.ExecTime)} = {profilerMetrics.ExecTime.Highest()}");
-            log.Log($"{nameof(profilerMetrics.ExecTimeWithChildren)} = {profilerMetrics.ExecTimeWithChildren.Highest()}");
-            log.Log($"{nameof(profilerMetrics.SingleExecTimeMax)} = {profilerMetrics.SingleExecTimeMax.Highest()}");
-            log.Log($"{nameof(profilerMetrics.WallTime)} = {profilerMetrics.WallTime.Highest()}");
-        }
-    }
-
-    public class AsyncProfileMetrics
-    {
-        public AsyncProfileMetrics(Metrics metrics)
-        {
-            CallCount = CreateMetric(metrics, "chronos_call_count_total");
-            ExecTime = CreateMetric(metrics, "chronos_exec_time_total");
-            ExecTimeWithChildren = CreateMetric(metrics, "chronos_exec_time_with_children_total");
-            SingleExecTimeMax = CreateMetric(metrics, "chronos_single_exec_time_max");
-            WallTime = CreateMetric(metrics, "chronos_wall_time_total");
-        }
-
-        public AsyncProfileMetric CallCount { get; }
-        public AsyncProfileMetric ExecTime { get; }
-        public AsyncProfileMetric ExecTimeWithChildren { get; }
-        public AsyncProfileMetric SingleExecTimeMax { get; }
-        public AsyncProfileMetric WallTime { get; }
-
-        private static AsyncProfileMetric CreateMetric(Metrics metrics, string name)
-        {
-            var sets = metrics.Sets.Where(s => s.Name == name).ToArray();
-            return new AsyncProfileMetric(sets);
-        }
-    }
-
-    public class AsyncProfileMetric
-    {
-        private readonly MetricsSet[] metricsSets;
-
-        public AsyncProfileMetric(MetricsSet[] metricsSets)
-        {
-            this.metricsSets = metricsSets;
-        }
-
-        public MetricsSet Highest()
-        {
-            MetricsSet? result = null;
-            var highest = double.MinValue;
-            foreach (var metric in metricsSets)
-            {
-                foreach (var value in metric.Values)
-                {
-                    if (value.Value > highest)
-                    {
-                        highest = value.Value;
-                        result = metric;
-                    }
-                }
-            }
-
-            if (result == null) throw new Exception("None were highest");
-            return result;
-        }
-    }
-}

Tests/CodexTests/BasicTests/EventLogginHandler.cs

@@ -0,0 +1,57 @@
+using CodexContractsPlugin.ChainMonitor;
+using GethPlugin;
+using Logging;
+using System.Numerics;
+
+namespace CodexTests.BasicTests
+{
+    public class EventLogginHandler : IChainStateChangeHandler
+    {
+        private readonly ILog log;
+
+        public EventLogginHandler(ILog log)
+        {
+            this.log = log;
+        }
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            Log(nameof(OnNewRequest), requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestCancelled), requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFailed), requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFinished), requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFulfilled), requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            Log(nameof(OnSlotFilled), requestEvent, host.ToString(), slotIndex.ToString());
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            Log(nameof(OnNewRequest), requestEvent, slotIndex.ToString());
+        }
+
+        private void Log(string name, object o, params string[] str)
+        {
+            log.Log(name + ": " + o.ToString() + " - " + string.Join(",", str));
+        }
+    }
+}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -2,6 +2,7 @@
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
+using Nethereum.JsonRpc.Client;
 using NUnit.Framework;
 using Utils;
 
@@ -10,6 +11,18 @@ namespace CodexTests.BasicTests
     [TestFixture]
     public class ExampleTests : CodexDistTest
     {
+        [Test]
+        public void A()
+        {
+            var oneMb = GenerateTestFile(1.MB(), "oneMB");
+            var fiveMb = GenerateTestFile(5.MB(), "fiveMb");
+            var tenMb = GenerateTestFile(10.MB(), "tenMb");
+            var hundredMb = GenerateTestFile(100.MB(), "hundredMb");
+            var oneGb = GenerateTestFile(1.GB(), "oneGb");
+
+            var a = 0;
+        }
+
         [Test]
         public void CodexLogExample()
         {
@@ -20,11 +33,41 @@ namespace CodexTests.BasicTests
             var localDatasets = primary.LocalFiles();
             CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
 
+            var nameMap = new Dictionary<string, string>();
+            AddNameMapping(nameMap, primary);
+
+            Get().Replacer = line =>
+            {
+                if (line == null) return null;
+                foreach (var pair in nameMap)
+                {
+                    line = line.Replace(pair.Key, pair.Value);
+                }
+                return line;
+            };
+
+
             var log = Ci.DownloadLog(primary);
 
             log.AssertLogContains("Uploaded file");
         }
 
+
+        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
+        {
+            var name = node.GetName();
+            var info = node.GetDebugInfo();
+            var nodeId = info.Table.LocalNode.NodeId;
+            var peerId = info.Table.LocalNode.PeerId;
+
+            nameMap.Add(nodeId, name);
+            nameMap.Add(peerId, name);
+            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToShortId(peerId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
+        }
+
         [Test]
         public void TwoMetricsExample()
         {
@@ -36,7 +79,7 @@ namespace CodexTests.BasicTests
             var primary2 = group2[0];
             var secondary2 = group2[1];
 
-            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(10), primary, primary2);
+            var metrics = Ci.GetMetricsFor(primary, primary2);
 
             primary.ConnectToPeer(secondary);
             primary2.ConnectToPeer(secondary2);

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
 using FileUtils;
@@ -34,10 +35,7 @@ namespace CodexTests.BasicTests
             var numberOfHosts = 5;
             var hosts = StartCodex(numberOfHosts, s => s
                 .WithName("Host")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                {
-                    ContractClock = CodexLogLevel.Trace,
-                })
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Info))
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), hostInitialBalance)
@@ -113,6 +111,104 @@ namespace CodexTests.BasicTests
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
         }
 
+        [Test]
+        [Combinatorial]
+        public void FindBug(
+            [Values(64)] int numBlocks,
+            [Values(0)] int plusSizeKb,
+            [Values(0)] int plusSizeBytes
+        )
+        {
+            var numberOfHosts = 15;
+
+            var hostInitialBalance = 234.Tst();
+            var clientInitialBalance = 100000.Tst();
+            var fileSize = new ByteSize(
+                numBlocks * (64 * 1024) +
+                plusSizeKb * 1024 +
+                plusSizeBytes
+            );
+
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var contracts = Ci.StartCodexContracts(geth);
+
+            var hosts = StartCodex(numberOfHosts, s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Trace))
+                .WithStorageQuota(11.GB())
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
+
+            foreach (var host in hosts)
+            {
+                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
+
+                var availability = new StorageAvailability(
+                    totalSpace: 10.GB(),
+                    maxDuration: TimeSpan.FromMinutes(30),
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 20.TstWei()
+                );
+                host.Marketplace.MakeStorageAvailable(availability);
+            }
+
+            var client = StartCodex(s => s
+                .WithName("Client")
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), clientInitialBalance)));
+
+            var nameMap = new Dictionary<string, string>();
+            AddNameMapping(nameMap, client);
+            foreach (var host in hosts) AddNameMapping(nameMap, host);
+
+            Get().Replacer = line =>
+            {
+                if (line == null) return null;
+                foreach (var pair in nameMap)
+                {
+                    line = line.Replace(pair.Key, pair.Value);
+                }
+                return line;
+            };
+
+            while (true)
+            {
+                var testFile = CreateFile(fileSize);
+                var uploadCid = client.UploadFile(testFile);
+
+                var purchase = new StoragePurchaseRequest(uploadCid)
+                {
+                    PricePerSlotPerSecond = 2.TstWei(),
+                    RequiredCollateral = 10.TstWei(),
+                    MinRequiredNumberOfNodes = 5,
+                    NodeFailureTolerance = 2,
+                    ProofProbability = 5,
+                    Duration = TimeSpan.FromMinutes(20),
+                    Expiry = TimeSpan.FromMinutes(10)
+                };
+
+                var purchaseContract = client.Marketplace.RequestStorage(purchase);
+                purchaseContract.WaitForStorageContractStarted();
+            }
+        }
+
+        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
+        {
+            var name = node.GetName();
+            var info = node.GetDebugInfo();
+            var nodeId = info.Table.LocalNode.NodeId;
+            var peerId = info.Table.LocalNode.PeerId;
+
+            nameMap.Add(nodeId, name);
+            nameMap.Add(peerId, name);
+            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToShortId(peerId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
+        }
+
         private TrackedFile CreateFile(ByteSize fileSize)
         {
             var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -1,7 +1,5 @@
 using CodexPlugin;
-using FileUtils;
 using NUnit.Framework;
-using System.Diagnostics;
 using Utils;
 
 namespace CodexTests.BasicTests
@@ -12,44 +10,11 @@ namespace CodexTests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var node = StartCodex();
+            var primary = StartCodex();
 
-            PerformOneClientTest(node);
+            PerformOneClientTest(primary);
 
-            LogNodeStatus(node);
-        }
-
-        [Test]
-        public void InterruptUploadTest()
-        {
-            var nodes = StartCodex(10);
-
-            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
-            Task.WaitAll(tasks.ToArray());
-
-            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
-        }
-
-        private bool RunInterruptUploadTest(ICodexNode node)
-        {
-            var file = GenerateTestFile(300.MB());
-
-            var process = StartCurlUploadProcess(node, file);
-
-            Thread.Sleep(500);
-            process.Kill();
-            Thread.Sleep(1000);
-
-            var log = Ci.DownloadLog(node);
-            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
-        }
-
-        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
-        {
-            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
-            var codexUrl = $"{apiAddress}/api/codex/v1/data";
-            var filePath = file.Filename;
-            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
+            LogNodeStatus(primary);
         }
 
         private void PerformOneClientTest(ICodexNode primary)

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -22,6 +22,26 @@ namespace CodexTests.BasicTests
             testFile.AssertIsEqual(downloadedFile);
         }
 
+        [Test]
+        public void FindBug()
+        {
+            var uploader = StartCodex();
+            var downloaders = StartCodex(10);
+
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(15))
+            {
+                var cid = uploader.UploadFile(GenerateTestFile(5.MB()));
+
+                var loop = Parallel.ForEach(downloaders, d =>
+                {
+                    d.DownloadContent(cid);
+                });
+
+                Assert.That(loop.IsCompleted);
+            }
+        }
+
         [Test]
         public void DownloadingUnknownCidDoesNotCauseCrash()
         {

Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs

@@ -1,228 +0,0 @@
-using CodexPlugin;
-using FileUtils;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.DownloadConnectivityTests
-{
-    [TestFixture]
-    public class MultiswarmTests : AutoBootstrapDistTest
-    {
-        [Test]
-        [Combinatorial]
-        public void Multiswarm(
-            [Values(3, 5)] int numFiles,
-            [Values(5, 20)] int fileSizeMb,
-            [Values(1)] int uploadersPerFile,
-            [Values(3)] int downloadersPerFile,
-            [Values(1)] int maxUploadsPerNode,
-            [Values(2, 3)] int maxDownloadsPerNode
-        )
-        {
-            var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
-            Assert.That(plan.NodePlans.Count, Is.LessThan(30));
-
-            RunThePlan(plan, fileSizeMb);
-        }
-
-        private void RunThePlan(Plan plan, int fileSizeMb)
-        {
-            foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB());
-            var nodes = StartCodex(plan.NodePlans.Count);
-            for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i];
-
-            // Upload all files to their nodes.
-            foreach (var filePlan in plan.FilePlans)
-            {
-                foreach (var uploader in filePlan.Uploaders)
-                {
-                    filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!);
-                }
-            }
-
-            Thread.Sleep(5000); // Everything is processed and announced.
-
-            // Start all downloads (almost) simultaneously.
-            var tasks = new List<Task>();
-            foreach (var filePlan in plan.FilePlans)
-            {
-                foreach (var downloader in filePlan.Downloaders)
-                {
-                    tasks.Add(Task.Run(() =>
-                    {
-                        var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!);
-                        lock (filePlan.DownloadedFiles)
-                        {
-                            filePlan.DownloadedFiles.Add(downloadedFile);
-                        }
-                    }));
-                }
-            }
-
-            Task.WaitAll(tasks.ToArray());
-
-            // Assert all files are correct.
-            foreach (var filePlan in plan.FilePlans)
-            {
-                foreach (var downloadedFile in filePlan.DownloadedFiles)
-                {
-                    filePlan.File!.AssertIsEqual(downloadedFile);
-                }
-            }
-        }
-
-        private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
-        {
-            var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
-            plan.Initialize();
-            plan.LogPlan(GetTestLog());
-            return plan;
-        }
-    }
-
-    public class FilePlan
-    {
-        public FilePlan(int number)
-        {
-            Number = number;
-        }
-
-        public int Number { get; }
-        public TrackedFile? File { get; set; }
-        public ContentId? Cid { get; set; }
-        public List<TrackedFile?> DownloadedFiles { get; } = new List<TrackedFile?>();
-        public List<NodePlan> Uploaders { get; } = new List<NodePlan>();
-        public List<NodePlan> Downloaders { get; } = new List<NodePlan>();
-
-        public override string ToString()
-        {
-            return $"FilePlan[{Number}] " +
-                $"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " +
-                $"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]";
-        }
-    }
-
-    public class NodePlan
-    {
-        public NodePlan(int number)
-        {
-            Number = number;
-        }
-
-        public int Number { get; }
-        public ICodexNode? Node { get; set; }
-        public List<FilePlan> Uploads { get; } = new List<FilePlan>();
-        public List<FilePlan> Downloads { get; } = new List<FilePlan>();
-
-        public bool Contains(FilePlan plan)
-        {
-            return Uploads.Contains(plan) || Downloads.Contains(plan);
-        }
-
-        public override string ToString()
-        {
-            return $"NodePlan[{Number}] " +
-                $"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " +
-                $"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]";
-        }
-    }
-
-    public class Plan
-    {
-        private readonly int numFiles;
-        private readonly int uploadersPerFile;
-        private readonly int downloadersPerFile;
-        private readonly int maxUploadsPerNode;
-        private readonly int maxDownloadsPerNode;
-
-        public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
-        {
-            this.numFiles = numFiles;
-            this.uploadersPerFile = uploadersPerFile;
-            this.downloadersPerFile = downloadersPerFile;
-            this.maxUploadsPerNode = maxUploadsPerNode;
-            this.maxDownloadsPerNode = maxDownloadsPerNode;
-        }
-
-        public List<FilePlan> FilePlans { get; } = new List<FilePlan>();
-        public List<NodePlan> NodePlans { get; } = new List<NodePlan>();
-
-        public void Initialize()
-        {
-            for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i));
-            foreach (var filePlan in FilePlans)
-            {
-                while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan);
-                while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan);
-            }
-
-            CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number));
-            CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number));
-
-            foreach (var filePlan in FilePlans)
-            {
-                Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile));
-                Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile));
-            }
-            foreach (var nodePlan in NodePlans)
-            {
-                Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode));
-                Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode));
-            }
-        }
-
-        public void LogPlan(ILog log)
-        {
-            log.Log("The plan:");
-            log.Log("Input:");
-            log.Log($"numFiles: {numFiles}");
-            log.Log($"uploadersPerFile: {uploadersPerFile}");
-            log.Log($"downloadersPerFile: {downloadersPerFile}");
-            log.Log($"maxUploadsPerNode: {maxUploadsPerNode}");
-            log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}");
-            log.Log("Setup:");
-            log.Log($"number of nodes: {NodePlans.Count}");
-            foreach (var filePlan in FilePlans) log.Log(filePlan.ToString());
-            foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString());
-        }
-
-        private void AddDownloader(FilePlan filePlan)
-        {
-            var nodePlan = GetOrCreateDownloaderNode(filePlan);
-            filePlan.Downloaders.Add(nodePlan);
-            nodePlan.Downloads.Add(filePlan);
-        }
-
-        private void AddUploader(FilePlan filePlan)
-        {
-            var nodePlan = GetOrCreateUploaderNode(filePlan);
-            filePlan.Uploaders.Add(nodePlan);
-            nodePlan.Uploads.Add(filePlan);
-        }
-
-        private NodePlan GetOrCreateDownloaderNode(FilePlan notIn)
-        {
-            var available = NodePlans.Where(n =>
-                n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
-            ).ToArray();
-            if (available.Any()) return RandomUtils.GetOneRandom(available);
-
-            var newNodePlan = new NodePlan(NodePlans.Count);
-            NodePlans.Add(newNodePlan);
-            return newNodePlan;
-        }
-
-        private NodePlan GetOrCreateUploaderNode(FilePlan notIn)
-        {
-            var available = NodePlans.Where(n =>
-                n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
-            ).ToArray();
-            if (available.Any()) return RandomUtils.GetOneRandom(available);
-
-            var newNodePlan = new NodePlan(NodePlans.Count);
-            NodePlans.Add(newNodePlan);
-            return newNodePlan;
-        }
-    }
-}

Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs

@@ -1,25 +0,0 @@
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.DownloadConnectivityTests
-{
-    [TestFixture]
-    public class SwarmTests : AutoBootstrapDistTest
-    {
-        [Test]
-        [Combinatorial]
-        [CreateTranscript("swarm_retransmit")]
-        public void DetectBlockRetransmits(
-            [Values(1, 5, 10, 20)] int fileSize,
-            [Values(3, 5, 10, 20)] int numNodes
-        )
-        {
-            var nodes = StartCodex(numNodes);
-            var file = GenerateTestFile(fileSize.MB());
-            var cid = nodes[0].UploadFile(file);
-
-            var tasks = nodes.Select(n => Task.Run(() => n.DownloadContent(cid))).ToArray();
-            Task.WaitAll(tasks);
-        }
-    }
-}

Tests/CodexTests/UtilityTests/DiscordBotTests.cs

@@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests
         private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
         private readonly List<ulong> rewardsSeen = new List<ulong>();
         private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
-        private readonly List<ChainEventMessage> receivedEvents = new List<ChainEventMessage>();
+        private readonly List<string> receivedEvents = new List<string>();
  
         [Test]
         [DontDownloadLogs]
@@ -73,18 +73,13 @@ namespace CodexTests.UtilityTests
 
         private void AssertEventOccurance(string msg, int expectedCount)
         {
-            Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount),
+            Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
                 $"Event '{msg}' did not occure correct number of times.");
         }
 
         private void OnCommand(string timestamp, GiveRewardsCommand call)
         {
             Log($"<API call {timestamp}>");
-            foreach (var e in call.EventsOverview)
-            {
-                Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order.");
-            }
-
             receivedEvents.AddRange(call.EventsOverview);
             foreach (var e in call.EventsOverview)
             {

Tests/DistTestCore/Configuration.cs

@@ -29,12 +29,12 @@ namespace DistTestCore
         /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace, Func<string?, string?> replacer)
         {
-            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
+            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace, replacer);
         }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace, Func<string?, string?> replacer)
         {
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
@@ -45,6 +45,7 @@ namespace DistTestCore
 
             config.AllowNamespaceOverride = false;
             config.Hooks = hooks;
+            config.Replacer = replacer;
 
             return config;
         }

Tests/DistTestCore/DistTest.cs

@@ -35,7 +35,7 @@ namespace DistTestCore
             fixtureLog = new FixtureLog(logConfig, startTime, deployId);
             statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
+            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix, s => s), configuration.GetFileManagerFolder());
 
             Initialize(fixtureLog);
         }

Tests/DistTestCore/TestLifecycle.cs

@@ -25,7 +25,7 @@ namespace DistTestCore
             TestNamespace = testNamespace;
             TestStart = DateTime.UtcNow;
 
-            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
+            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace, InternalReplacer), configuration.GetFileManagerFolder(), timeSet);
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
             this.deployId = deployId;
@@ -33,6 +33,11 @@ namespace DistTestCore
             log.WriteLogTag();
         }
 
+        private string? InternalReplacer(string? arg)
+        {
+            return Replacer(arg);
+        }
+
         public DateTime TestStart { get; }
         public TestLog Log { get; }
         public Configuration Configuration { get; }
@@ -40,6 +45,7 @@ namespace DistTestCore
         public string TestNamespace { get; }
         public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
+        public Func<string?, string?> Replacer { get; set; } = s => s;
 
         public void DeleteAllResources()
         {

Tests/FrameworkTests/FrameworkTests.csproj

@@ -18,7 +18,6 @@
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
-    <ProjectReference Include="..\..\Tools\TestNetRewarder\TestNetRewarder.csproj" />
   </ItemGroup>
 
 </Project>

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -19,20 +19,11 @@ namespace FrameworkTests.NethereumWorkflow
         {
             var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30);
             blocks = new Dictionary<ulong, Block>();
-
-            Block? prev = null;
+            
             for (ulong i = 0; i < 30; i++)
             {
                 ulong d = 100 + i;
-                var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2));
-                blocks.Add(d, newBlock);
-
-                if (prev != null)
-                {
-                    prev.Next = newBlock;
-                    newBlock.Previous = prev;
-                }
-                prev = newBlock;
+                blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2)));
             }
         }
 
@@ -108,23 +99,23 @@ namespace FrameworkTests.NethereumWorkflow
         }
 
         [Test]
-        public void FindsGenesisBlockAtFrontOfChain()
+        public void FailsToFindBlockBeforeFrontOfChain()
         {
             var first = blocks.First().Value;
 
-            var firstNumber = finder.GetHighestBlockNumberBefore(first.Time);
+            var notFound = finder.GetHighestBlockNumberBefore(first.Time);
 
-            Assert.That(firstNumber, Is.EqualTo(first.Number));
+            Assert.That(notFound, Is.Null);
         }
 
         [Test]
-        public void FindsCurrentBlockAtTailOfChain()
+        public void FailsToFindBlockAfterTailOfChain()
         {
             var last = blocks.Last().Value;
 
-            var lastNumber = finder.GetLowestBlockNumberAfter(last.Time);
+            var notFound = finder.GetLowestBlockNumberAfter(last.Time);
 
-            Assert.That(lastNumber, Is.EqualTo(last.Number));
+            Assert.That(notFound, Is.Null);
         }
 
         [Test]
@@ -152,27 +143,13 @@ namespace FrameworkTests.NethereumWorkflow
         {
             foreach (var pair in blocks)
             {
-                var block = pair.Value;
+                finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
+                finder.GetHighestBlockNumberBefore(pair.Value.Time);
+                finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
 
-                AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore));
-                AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time));
-                AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter));
-
-                AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore));
-                AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time));
-                AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter));
-            }
-        }
-
-        private void AssertLink(Block? expected, ulong? actual)
-        {
-            if (expected == null)
-            {
-                Assert.That(actual, Is.Null);
-            }
-            else
-            {
-                Assert.That(expected.Number, Is.EqualTo(actual!.Value));
+                finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
+                finder.GetLowestBlockNumberAfter(pair.Value.Time);
+                finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
             }
         }
     }
@@ -190,9 +167,6 @@ namespace FrameworkTests.NethereumWorkflow
         public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
         public DateTime JustAfter { get { return Time.AddSeconds(1); } }
 
-        public Block? Next { get; set; }
-        public Block? Previous { get; set; }
-
         public override string ToString()
         {
             return $"[{Number}]";

Tests/FrameworkTests/OverwatchTranscript/TranscriptLargeTest.cs

@@ -2,7 +2,7 @@
 using NUnit.Framework;
 using OverwatchTranscript;
 
-namespace FrameworkTests.OverwatchTranscriptTests
+namespace FrameworkTests.OverwatchTranscript
 {
     [TestFixture]
     public class TranscriptLargeTests

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -0,0 +1,136 @@
+using Logging;
+using NUnit.Framework;
+using OverwatchTranscript;
+
+namespace FrameworkTests.OverwatchTranscript
+{
+    [TestFixture]
+    public class TranscriptTests
+    {
+        private const string TranscriptFilename = "testtranscript.owts";
+        private const string HeaderKey = "testHeader";
+        private const string HeaderData = "abcdef";
+        private const string EventData0 = "12345";
+        private const string EventData1 = "678";
+        private const string EventData2 = "90";
+        private const string EventData3 = "-=";
+        private readonly DateTime t0 = DateTime.UtcNow;
+        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
+        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
+
+        [Test]
+        public void WriteAndRun()
+        {
+            // unstable.
+            WriteTranscript();
+            ReadTranscript();
+
+            File.Delete(TranscriptFilename);
+        }
+
+        private void WriteTranscript()
+        {
+            var log = new ConsoleLog();
+            var writer = Transcript.NewWriter(log);
+
+            writer.AddHeader(HeaderKey, new TestHeader
+            {
+                HeaderData = HeaderData
+            });
+
+            writer.Add(t0, new MyEvent
+            {
+                EventData = EventData0
+            });
+            writer.Add(t2, new MyEvent
+            {
+                EventData = EventData3
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData1
+            });
+            writer.Add(t1, new MyEvent
+            {
+                EventData = EventData2
+            });
+
+            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
+
+            writer.Write(TranscriptFilename);
+        }
+
+        private void ReadTranscript()
+        {
+            var reader = Transcript.NewReader(TranscriptFilename);
+
+            var header = reader.GetHeader<TestHeader>(HeaderKey);
+            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
+            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
+            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
+            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
+            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
+
+            var moments = new List<ActivateMoment>();
+            var events = new List<ActivateEvent<MyEvent>>();
+            reader.AddMomentHandler(moments.Add);
+            reader.AddEventHandler<MyEvent>(events.Add);
+
+
+            Assert.That(moments.Count, Is.EqualTo(0));
+            Assert.That(events.Count, Is.EqualTo(0));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(1));
+            Assert.That(events.Count, Is.EqualTo(1));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(2));
+            Assert.That(events.Count, Is.EqualTo(3));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            reader.Next();
+            Assert.That(moments.Count, Is.EqualTo(3));
+            Assert.That(events.Count, Is.EqualTo(4));
+
+            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
+            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
+            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
+
+            AssertEvent(events[0], utc: t0, duration: t1 - t0, index: 0, data: EventData0);
+            AssertEvent(events[1], utc: t1, duration: t2 - t1, index: 1, data: EventData1);
+            AssertEvent(events[2], utc: t1, duration: t2 - t1, index: 1, data: EventData2);
+            AssertEvent(events[3], utc: t2, duration: null, index: 2, data: EventData3);
+
+            reader.Close();
+        }
+
+        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
+        {
+            Assert.That(m.Utc, Is.EqualTo(utc));
+            Assert.That(m.Duration, Is.EqualTo(duration));
+            Assert.That(m.Index, Is.EqualTo(index));
+        }
+
+        private void AssertEvent(ActivateEvent<MyEvent> e, DateTime utc, TimeSpan? duration, int index, string data)
+        {
+            Assert.That(e.Moment.Utc, Is.EqualTo(utc));
+            Assert.That(e.Moment.Duration, Is.EqualTo(duration));
+            Assert.That(e.Moment.Index, Is.EqualTo(index));
+            Assert.That(e.Payload.EventData, Is.EqualTo(data));
+        }
+    }
+
+    public class TestHeader
+    {
+        public string HeaderData { get; set; } = string.Empty;
+    }
+
+    public class MyEvent
+    {
+        public string EventData { get; set; } = string.Empty;
+    }
+}

Tests/FrameworkTests/OverwatchTranscriptTests/TranscriptTests.cs

@@ -1,229 +0,0 @@
-using Logging;
-using Newtonsoft.Json;
-using NUnit.Framework;
-using OverwatchTranscript;
-using System.IO.Compression;
-
-namespace FrameworkTests.OverwatchTranscriptTests
-{
-    [TestFixture]
-    public class TranscriptTests
-    {
-        private const string TranscriptFilename = "testtranscript.owts";
-        private const string HeaderKey = "testHeader";
-        private const string HeaderData = "abcdef";
-        private const string EventData0 = "12345";
-        private const string EventData1 = "678";
-        private const string EventData2 = "90";
-        private const string EventData3 = "-=";
-        private readonly DateTime t0 = DateTime.UtcNow;
-        private readonly DateTime t1 = DateTime.UtcNow.AddMinutes(1);
-        private readonly DateTime t2 = DateTime.UtcNow.AddMinutes(3);
-
-        [Test]
-        [Combinatorial]
-        public void WriteAndRun()
-        {
-            WriteTranscript();
-            AssertFileContent();
-            ReadTranscript();
-
-            File.Delete(TranscriptFilename);
-        }
-
-        private void WriteTranscript()
-        {
-            var log = new ConsoleLog();
-            var writer = Transcript.NewWriter(log);
-
-            writer.AddHeader(HeaderKey, new TestHeader
-            {
-                HeaderData = HeaderData
-            });
-
-            writer.Add(t0, new MyEvent
-            {
-                EventData = EventData0
-            });
-            writer.Add(t2, new MyEvent
-            {
-                EventData = EventData3
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData1
-            });
-            writer.Add(t1, new MyEvent
-            {
-                EventData = EventData2
-            });
-
-            if (File.Exists(TranscriptFilename)) File.Delete(TranscriptFilename);
-
-            writer.Write(TranscriptFilename);
-        }
-
-        private void ReadTranscript()
-        {
-            var reader = Transcript.NewReader(TranscriptFilename);
-
-            var header = reader.GetHeader<TestHeader>(HeaderKey);
-            Assert.That(header.HeaderData, Is.EqualTo(HeaderData));
-            Assert.That(reader.Header.NumberOfMoments, Is.EqualTo(3));
-            Assert.That(reader.Header.NumberOfEvents, Is.EqualTo(4));
-            Assert.That(reader.Header.EarliestUtc, Is.EqualTo(t0));
-            Assert.That(reader.Header.LatestUtc, Is.EqualTo(t2));
-
-            var moments = new List<ActivateMoment>();
-            var events = new List<ActivateEvent<MyEvent>>();
-            reader.AddMomentHandler(moments.Add);
-            reader.AddEventHandler<MyEvent>(events.Add);
-
-            var timeout = 10;
-            while (moments.Count < 3 && events.Count < 4)
-            {
-                Thread.Sleep(10);
-                reader.Next();
-
-                timeout--;
-                if (timeout == 0) Assert.Fail("Events not received.");
-            }
-
-            reader.Next();
-            Assert.That(moments.Count, Is.EqualTo(3));
-            Assert.That(events.Count, Is.EqualTo(4));
-
-            AssertMoment(moments[0], utc: t0, duration: t1 - t0, index: 0);
-            AssertMoment(moments[1], utc: t1, duration: t2 - t1, index: 1);
-            AssertMoment(moments[2], utc: t2, duration: null, index: 2);
-
-            AssertEvent(events, utc: t0, duration: t1 - t0, index: 0, data: EventData0);
-            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData2);
-            AssertEvent(events, utc: t1, duration: t2 - t1, index: 1, data: EventData1);
-            AssertEvent(events, utc: t2, duration: null, index: 2, data: EventData3);
-
-            reader.Close();
-        }
-
-        private void AssertMoment(ActivateMoment m, DateTime utc, TimeSpan? duration, int index)
-        {
-            Assert.That(m.Utc, Is.EqualTo(utc));
-            Assert.That(m.Duration, Is.EqualTo(duration));
-            Assert.That(m.Index, Is.EqualTo(index));
-        }
-
-        private void AssertEvent(List<ActivateEvent<MyEvent>> events, DateTime utc, TimeSpan? duration, int index, string data)
-        {
-            var e = events.SingleOrDefault(e => e.Moment.Utc == utc && e.Payload.EventData == data);
-            if (e == null) Assert.Fail("Event not found");
-
-            Assert.That(e!.Moment.Utc, Is.EqualTo(utc));
-            Assert.That(e!.Moment.Duration, Is.EqualTo(duration));
-            Assert.That(e!.Moment.Index, Is.EqualTo(index));
-            Assert.That(e!.Payload.EventData, Is.EqualTo(data));
-        }
-
-        private void AssertFileContent()
-        {
-            using var zip = ZipFile.OpenRead(TranscriptFilename);
-            Assert.That(zip.Entries.Count, Is.EqualTo(2));
-            foreach (var entry in zip.Entries)
-            {
-                if (entry.Name == "transcript.json")
-                {
-                    var transcript = ZipEntryJson<OverwatchTranscript.OverwatchTranscript>(entry);
-                    AssertTranscript(transcript);
-                }
-                else
-                {
-                    var moments = ZipEntryToMoments(entry);
-                    AssertMoments(moments);
-                }
-            }
-        }
-
-        private void AssertTranscript(OverwatchTranscript.OverwatchTranscript transcript)
-        {
-            Assert.That(transcript.Header.Common.NumberOfMoments, Is.EqualTo(3));
-            Assert.That(transcript.Header.Common.NumberOfEvents, Is.EqualTo(4));
-            Assert.That(transcript.Header.Common.EarliestUtc, Is.EqualTo(t0));
-            Assert.That(transcript.Header.Common.LatestUtc, Is.EqualTo(t2));
-            Assert.That(transcript.Header.Entries.Length, Is.EqualTo(1));
-            Assert.That(transcript.Header.Entries[0].Key, Is.EqualTo(HeaderKey));
-            Assert.That(transcript.Header.Entries[0].Value, Is.EqualTo("{\"HeaderData\":\"abcdef\"}"));
-
-            Assert.That(transcript.MomentReferences.Length, Is.EqualTo(1));
-            Assert.That(transcript.MomentReferences[0].NumberOfMoments, Is.EqualTo(3));
-            Assert.That(transcript.MomentReferences[0].NumberOfEvents, Is.EqualTo(4));
-            Assert.That(transcript.MomentReferences[0].EarliestUtc, Is.EqualTo(t0));
-            Assert.That(transcript.MomentReferences[0].LatestUtc, Is.EqualTo(t2));
-        }
-
-        private void AssertMoments(OverwatchMoment[] moments)
-        {
-            Assert.That(moments.Length, Is.EqualTo(3));
-
-            Assert.That(moments[0].Utc, Is.EqualTo(t0));
-            Assert.That(moments[0].Events.Length, Is.EqualTo(1));
-            Assert.That(moments[0].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
-            Assert.That(moments[0].Events[0].Payload, Is.EqualTo("{\"EventData\":\"12345\"}"));
-
-            Assert.That(moments[1].Utc, Is.EqualTo(t1));
-            Assert.That(moments[1].Events.Length, Is.EqualTo(2));
-            Assert.That(moments[1].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
-            Assert.That(moments[1].Events[1].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
-
-            // output order is not guaranteed:
-            var payloads = moments[1].Events.Select(e => e.Payload).ToArray();
-            CollectionAssert.AreEquivalent(new[]
-            {
-                "{\"EventData\":\"90\"}",
-                "{\"EventData\":\"678\"}"
-            }, payloads);
-
-            Assert.That(moments[2].Utc, Is.EqualTo(t2));
-            Assert.That(moments[2].Events.Length, Is.EqualTo(1));
-            Assert.That(moments[2].Events[0].Type, Is.EqualTo("FrameworkTests.OverwatchTranscriptTests.MyEvent"));
-            Assert.That(moments[2].Events[0].Payload, Is.EqualTo("{\"EventData\":\"-=\"}"));
-        }
-
-        private T ZipEntryJson<T>(ZipArchiveEntry? entry)
-        {
-            if (entry == null) Assert.Fail("entry is null");
-            using var stream = entry!.Open();
-            using var reader = new StreamReader(stream);
-            var json = reader.ReadToEnd();
-            var result = JsonConvert.DeserializeObject<T>(json);
-            if (result == null) Assert.Fail("didn't deserialize");
-            return result!;
-        }
-
-        private OverwatchMoment[] ZipEntryToMoments(ZipArchiveEntry? entry)
-        {
-            var result = new List<OverwatchMoment>();
-            if (entry == null) Assert.Fail("entry is null");
-            using var stream = entry!.Open();
-            using var reader = new StreamReader(stream);
-
-            var line = reader.ReadLine();
-            while (!string.IsNullOrEmpty(line))
-            {
-                var moment = JsonConvert.DeserializeObject<OverwatchMoment>(line);
-                if (moment != null) result.Add(moment);
-                line = reader.ReadLine();
-            }
-
-            return result.ToArray();
-        }
-    }
-
-    public class TestHeader
-    {
-        public string HeaderData { get; set; } = string.Empty;
-    }
-
-    public class MyEvent
-    {
-        public string EventData { get; set; } = string.Empty;
-    }
-}

Tests/FrameworkTests/Utils/EmojiMapsTests.cs

@@ -1,28 +0,0 @@
-using NUnit.Framework;
-using System.Text;
-using TestNetRewarder;
-
-namespace FrameworkTests.Utils
-{
-    [TestFixture]
-    public class EmojiMapsTests
-    {
-        private readonly Random random = new Random();
-        private readonly EmojiMaps maps = new EmojiMaps();
-
-        [Test]
-        public void GeneratesConsistentStrings(
-            [Values(1, 5, 10, 20)] int inputLength,
-            [Values(1, 2, 3, 5)] int outLength)
-        {
-            var buffer = new byte[inputLength];
-            random.NextBytes(buffer);
-            var input = Encoding.ASCII.GetString(buffer);
-
-            var out1 = maps.StringToEmojis(input, outLength);
-            var out2 = maps.StringToEmojis(input, outLength);
-            
-            Assert.That(out1, Is.EqualTo(out2));
-        }
-    }
-}

Tests/FrameworkTests/Utils/RunLengthEncoding.cs

@@ -1,334 +0,0 @@
-namespace FrameworkTests.Utils
-{
-    public class Run
-    {
-        public Run(int start, int length)
-        {
-            Start = start;
-            Length = length;
-        }
-
-        public int Start { get; }
-        public int Length { get; private set; }
-
-        public bool Includes(int index)
-        {
-            return index >= Start && index < (Start + Length);
-        }
-
-        public RunUpdate ExpandToInclude(int index)
-        {
-            if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}");
-            if (index == (Start + Length))
-            {
-                Length++;
-                return new RunUpdate();
-            }
-            if (index == (Start - 1))
-            {
-                return new RunUpdate(
-                    newRuns: [new Run(Start - 1, Length + 1)],
-                    removeRuns: [this]
-                );
-            }
-            throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}");
-        }
-
-        public RunUpdate Unset(int index)
-        {
-            if (!Includes(index))
-            {
-                return new RunUpdate();
-            }
-
-            if (index == Start)
-            {
-                // First index: Replace self with new run at next index, unless empty.
-                if (Length == 1)
-                {
-                    return new RunUpdate(
-                        newRuns: Array.Empty<Run>(),
-                        removeRuns: [this]
-                    );
-                }
-                return new RunUpdate(
-                    newRuns: [new Run(Start + 1, Length - 1)],
-                    removeRuns: [this]
-                );
-            }
-
-            if (index == (Start + Length - 1))
-            {
-                // Last index: Become one smaller.
-                Length--;
-                return new RunUpdate();
-            }
-
-            // Split:
-            var newRunLength = (Start + Length - 1) - index;
-            Length = index - Start;
-            return new RunUpdate(
-                newRuns: [new Run(index + 1, newRunLength)],
-                removeRuns: Array.Empty<Run>()
-            );
-        }
-
-        public void Iterate(Action<int> action)
-        {
-            for (var i = 0; i < Length; i++)
-            {
-                action(Start + i);
-            }
-        }
-
-        public override string ToString()
-        {
-            return $"[{Start},{Length}]";
-        }
-
-        public override bool Equals(object? obj)
-        {
-            return obj is Run run &&
-                   Start == run.Start &&
-                   Length == run.Length;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(Start, Length);
-        }
-
-        public static bool operator ==(Run? obj1, Run? obj2)
-        {
-            if (ReferenceEquals(obj1, obj2)) return true;
-            if (ReferenceEquals(obj1, null)) return false;
-            if (ReferenceEquals(obj2, null)) return false;
-            return obj1.Equals(obj2);
-        }
-        public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2);
-    }
-
-    public class RunUpdate
-    {
-        public RunUpdate()
-            : this(Array.Empty<Run>(), Array.Empty<Run>())
-        {
-        }
-
-        public RunUpdate(Run[] newRuns, Run[] removeRuns)
-        {
-            NewRuns = newRuns;
-            RemoveRuns = removeRuns;
-        }
-
-        public Run[] NewRuns { get; }
-        public Run[] RemoveRuns { get; }
-    }
-
-    public partial class IndexSet
-    {
-        private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
-
-        public IndexSet()
-        {
-        }
-
-        public IndexSet(int[] indices)
-        {
-            foreach (var i in indices) Set(i);
-        }
-
-        public static IndexSet FromRunLengthEncoded(int[] rle)
-        {
-            var set = new IndexSet();
-            for (var i = 0; i < rle.Length; i += 2)
-            {
-                var start = rle[i];
-                var length = rle[i + 1];
-                set.runs.Add(start, new Run(start, length));
-            }
-
-            return set;
-        }
-
-        public bool IsSet(int index)
-        {
-            if (runs.ContainsKey(index)) return true;
-
-            var run = GetRunAt(index);
-            if (run == null) return false;
-            return true;
-        }
-
-        public void Set(int index)
-        {
-            if (IsSet(index)) return;
-
-            var runBefore = GetRunAt(index - 1);
-            var runAfter = GetRunExact(index + 1);
-
-            if (runBefore == null)
-            {
-                if (runAfter == null)
-                {
-                    CreateNewRun(index);
-                }
-                else
-                {
-                    HandleUpdate(runAfter.ExpandToInclude(index));
-                }
-            }
-            else
-            {
-                if (runAfter == null)
-                {
-                    HandleUpdate(runBefore.ExpandToInclude(index));
-                }
-                else
-                {
-                    // new index will connect runBefore with runAfter. We merge!
-                    HandleUpdate(new RunUpdate(
-                        newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)],
-                        removeRuns: [runBefore, runAfter]
-                    ));
-                }
-            }
-        }
-
-        public void Unset(int index)
-        {
-            if (runs.ContainsKey(index))
-            {
-                HandleUpdate(runs[index].Unset(index));
-            }
-            else
-            {
-                var run = GetRunAt(index);
-                if (run == null) return;
-                HandleUpdate(run.Unset(index));
-            }
-        }
-
-        public void Iterate(Action<int> onIndex)
-        {
-            foreach (var run in runs.Values)
-            {
-                run.Iterate(onIndex);
-            }
-        }
-
-        public int[] RunLengthEncoded()
-        {
-            return Encode().ToArray();
-        }
-
-        public override string ToString()
-        {
-            return string.Join("&", runs.Select(r => r.ToString()).ToArray());
-        }
-
-        private IEnumerable<int> Encode()
-        {
-            foreach (var pair in runs)
-            {
-                yield return pair.Value.Start;
-                yield return pair.Value.Length;
-            }
-        }
-
-        private Run? GetRunAt(int index)
-        {
-            foreach (var run in runs.Values)
-            {
-                if (run.Includes(index)) return run;
-            }
-            return null;
-        }
-
-        private Run? GetRunExact(int index)
-        {
-            if (runs.ContainsKey(index)) return runs[index];
-            return null;
-        }
-
-        private void HandleUpdate(RunUpdate runUpdate)
-        {
-            foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
-            foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
-        }
-
-        private void CreateNewRun(int index)
-        {
-            if (runs.ContainsKey(index + 1))
-            {
-                var length = runs[index + 1].Length + 1;
-                runs.Add(index, new Run(index, length));
-                runs.Remove(index + 1);
-            }
-            else
-            {
-                runs.Add(index, new Run(index, 1));
-            }
-        }
-    }
-
-    public partial class IndexSet
-    {
-        public IndexSet Overlap(IndexSet other)
-        {
-            var result = new IndexSet();
-            Iterate(i =>
-            {
-                if (other.IsSet(i)) result.Set(i);
-            });
-            return result;
-        }
-
-        public IndexSet Merge(IndexSet other)
-        {
-            var result = new IndexSet();
-            Iterate(result.Set);
-            other.Iterate(result.Set);
-            return result;
-        }
-
-        public IndexSet Without(IndexSet other)
-        {
-            var result = new IndexSet();
-            Iterate(i =>
-            {
-                if (!other.IsSet(i)) result.Set(i);
-            });
-            return result;
-        }
-
-        public override bool Equals(object? obj)
-        {
-            if (obj is IndexSet set)
-            {
-                if (set.runs.Count != runs.Count) return false;
-                foreach (var pair in runs)
-                {
-                    if (!set.runs.ContainsKey(pair.Key)) return false;
-                    if (set.runs[pair.Key] != pair.Value) return false;
-                }
-                return true;
-            }
-            return false;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(runs);
-        }
-
-        public static bool operator ==(IndexSet? obj1, IndexSet? obj2)
-        {
-            if (ReferenceEquals(obj1, obj2)) return true;
-            if (ReferenceEquals(obj1, null)) return false;
-            if (ReferenceEquals(obj2, null)) return false;
-            return obj1.Equals(obj2);
-        }
-        public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2);
-    }
-}

Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs

@@ -1,94 +0,0 @@
-using NUnit.Framework;
-
-namespace FrameworkTests.Utils
-{
-    [TestFixture]
-    public class RunLengthEncodingLogicalTests
-    {
-        [Test]
-        public void EqualityTest()
-        {
-            var setA = new IndexSet([1, 2, 3, 4]);
-            var setB = new IndexSet([1, 2, 3, 4]);
-
-            Assert.That(setA, Is.EqualTo(setB));
-            Assert.That(setA == setB);
-        }
-
-        [Test]
-        public void InequalityTest1()
-        {
-            var setA = new IndexSet([1, 2, 4, 5]);
-            var setB = new IndexSet([1, 2, 3, 4]);
-
-            Assert.That(setA, Is.Not.EqualTo(setB));
-            Assert.That(setA != setB);
-        }
-
-        [Test]
-        public void InequalityTest2()
-        {
-            var setA = new IndexSet([1, 2, 3]);
-            var setB = new IndexSet([1, 2, 3, 4]);
-
-            Assert.That(setA, Is.Not.EqualTo(setB));
-            Assert.That(setA != setB);
-        }
-
-        [Test]
-        public void InequalityTest3()
-        {
-            var setA = new IndexSet([2, 3, 4, 5]);
-            var setB = new IndexSet([1, 2, 3, 4]);
-
-            Assert.That(setA, Is.Not.EqualTo(setB));
-            Assert.That(setA != setB);
-        }
-
-        [Test]
-        public void InequalityTest()
-        {
-            var setA = new IndexSet([2, 3, 4]);
-            var setB = new IndexSet([1, 2, 3, 4]);
-
-            Assert.That(setA, Is.Not.EqualTo(setB));
-            Assert.That(setA != setB);
-        }
-
-        [Test]
-        public void Overlap()
-        {
-            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
-            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
-            var expectedSet = new IndexSet([3, 4, 5, 11]);
-
-            var set = setA.Overlap(setB);
-
-            Assert.That(set, Is.EqualTo(expectedSet));
-        }
-
-        [Test]
-        public void Merge()
-        {
-            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
-            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
-            var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]);
-
-            var set = setA.Merge(setB);
-
-            Assert.That(set, Is.EqualTo(expectedSet));
-        }
-
-        [Test]
-        public void Without()
-        {
-            var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
-            var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
-            var expectedSet = new IndexSet([1, 2, 14]);
-
-            var set = setA.Without(setB);
-
-            Assert.That(set, Is.EqualTo(expectedSet));
-        }
-    }
-}

Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs

@@ -1,40 +1,12 @@
 using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using static FrameworkTests.Utils.RunLengthEncodingTests;
 
 namespace FrameworkTests.Utils
 {
     [TestFixture]
     public class RunLengthEncodingRunTests
     {
-        [Test]
-        public void EqualityTest()
-        {
-            var runA = new Run(1, 4);
-            var runB = new Run(1, 4);
-
-            Assert.That(runA, Is.EqualTo(runB));
-            Assert.That(runA == runB);
-        }
-
-        [Test]
-        public void InequalityTest1()
-        {
-            var runA = new Run(1, 4);
-            var runB = new Run(1, 5);
-
-            Assert.That(runA, Is.Not.EqualTo(runB));
-            Assert.That(runA != runB);
-        }
-
-        [Test]
-        public void InequalityTest2()
-        {
-            var runA = new Run(1, 4);
-            var runB = new Run(2, 4);
-
-            Assert.That(runA, Is.Not.EqualTo(runB));
-            Assert.That(runA != runB);
-        }
-
         [Test]
         [Combinatorial]
         public void RunIncludes(
@@ -61,58 +33,23 @@ namespace FrameworkTests.Utils
         }
 
         [Test]
-        public void RunExpandThrowsWhenIndexNotAdjacent()
+        public void RunExpandToInclude()
         {
             var run = new Run(2, 3);
-            Assert.That(!run.Includes(1));
             Assert.That(run.Includes(2));
             Assert.That(run.Includes(4));
             Assert.That(!run.Includes(5));
 
-            Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf<Exception>());
-            Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf<Exception>());
-        }
+            Assert.That(run.ExpandToInclude(1), Is.False);
+            Assert.That(run.ExpandToInclude(2), Is.False);
+            Assert.That(run.ExpandToInclude(4), Is.False);
+            Assert.That(run.ExpandToInclude(6), Is.False);
 
-        [Test]
-        public void RunExpandThrowsWhenIndexAlreadyIncluded()
-        {
-            var run = new Run(2, 3);
-            Assert.That(!run.Includes(1));
-            Assert.That(run.Includes(2));
-            Assert.That(run.Includes(4));
-            Assert.That(!run.Includes(5));
-
-            Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf<Exception>());
-            Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf<Exception>());
-        }
-
-        [Test]
-        public void RunExpandToIncludeAfter()
-        {
-            var run = new Run(2, 3);
-            var update = run.ExpandToInclude(5);
-            Assert.That(update, Is.Not.Null);
-            Assert.That(update.NewRuns.Length, Is.EqualTo(0));
-            Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
+            Assert.That(run.ExpandToInclude(5), Is.True);
             Assert.That(run.Includes(5));
             Assert.That(!run.Includes(6));
         }
 
-        [Test]
-        public void RunExpandToIncludeBefore()
-        {
-            var run = new Run(2, 3);
-            var update = run.ExpandToInclude(1);
-
-            Assert.That(update, Is.Not.Null);
-            Assert.That(update.NewRuns.Length, Is.EqualTo(1));
-            Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
-
-            Assert.That(update.RemoveRuns[0], Is.SameAs(run));
-            Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
-            Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
-        }
-
         [Test]
         public void RunCanUnsetLastIndex()
         {
@@ -163,9 +100,94 @@ namespace FrameworkTests.Utils
         {
             var run = new Run(2, 4);
             var seen = new List<int>();
-            run.Iterate(seen.Add);
+            run.Iterate(i => seen.Add(i));
 
             CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
         }
     }
+
+    public class Run
+    {
+        public Run(int start, int length)
+        {
+            Start = start;
+            Length = length;
+        }
+
+        public int Start { get; }
+        public int Length { get; private set; }
+
+        public bool Includes(int index)
+        {
+            return index >= Start && index < (Start + Length);
+        }
+
+        public bool ExpandToInclude(int index)
+        {
+            if (index == (Start + Length))
+            {
+                Length++;
+                return true;
+            }
+            return false;
+        }
+
+        public RunUpdate Unset(int index)
+        {
+            if (!Includes(index))
+            {
+                return new RunUpdate();
+            }
+
+            if (index == Start)
+            {
+                // First index: Replace self with new run at next index, unless empty.
+                if (Length == 1)
+                {
+                    return new RunUpdate(Array.Empty<Run>(), new[] { this });
+                }
+                return new RunUpdate(
+                    newRuns: new[] { new Run(Start + 1, Length - 1) },
+                    removeRuns: new[] { this }
+                );
+            }
+
+            if (index == (Start + Length - 1))
+            {
+                // Last index: Become one smaller.
+                Length--;
+                return new RunUpdate();
+            }
+
+            // Split:
+            var newRunLength = (Start + Length - 1) - index;
+            Length = index - Start;
+            return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
+        }
+        
+        public void Iterate(Action<int> action)
+        {
+            for (var i = 0; i < Length; i++)
+            {
+                action(Start + i);
+            }
+        }
+    }
+
+    public class RunUpdate
+    {
+        public RunUpdate()
+            : this(Array.Empty<Run>(), Array.Empty<Run>())
+        {
+        }
+
+        public RunUpdate(Run[] newRuns, Run[] removeRuns)
+        {
+            NewRuns = newRuns;
+            RemoveRuns = removeRuns;
+        }
+
+        public Run[] NewRuns { get; }
+        public Run[] RemoveRuns { get; }
+    }
 }

Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs

@@ -1,4 +1,9 @@
-using NUnit.Framework;
+using Logging;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using NuGet.Frameworks;
+using NUnit.Framework;
+using System.Collections.Concurrent;
+using System.Numerics;
 using Utils;
 
 namespace FrameworkTests.Utils
@@ -114,19 +119,6 @@ namespace FrameworkTests.Utils
             }, encoded);
         }
 
-        [Test]
-        public void SetIndexBetweenRuns()
-        {
-            var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 });
-            set.Set(11);
-            var encoded = set.RunLengthEncoded();
-
-            CollectionAssert.AreEqual(new[]
-            {
-                8, 7
-            }, encoded);
-        }
-
         [Test]
         public void SetIndexAfterRun()
         {
@@ -209,5 +201,120 @@ namespace FrameworkTests.Utils
             all.Sort();
             return all.ToArray();
         }
+
+        public class IndexSet
+        {
+            private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
+
+            public IndexSet()
+            {
+            }
+
+            public IndexSet(int[] indices)
+            {
+                foreach (var i in indices) Set(i);
+            }
+
+            public static IndexSet FromRunLengthEncoded(int[] rle)
+            {
+                var set = new IndexSet();
+                for (var i = 0; i < rle.Length; i += 2)
+                {
+                    var start = rle[i];
+                    var length = rle[i + 1];
+                    set.runs.Add(start, new Run(start, length));
+                }
+
+                return set;
+            }
+
+            public bool IsSet(int index)
+            {
+                if (runs.ContainsKey(index)) return true;
+
+                var run = GetRunBefore(index);
+                if (run == null) return false;
+
+                return run.Includes(index);
+            }
+
+            public void Set(int index)
+            {
+                if (runs.ContainsKey(index)) return;
+
+                var run = GetRunBefore(index);
+                if (run == null || !run.ExpandToInclude(index))
+                {
+                    CreateNewRun(index);
+                }
+            }
+
+            public void Unset(int index)
+            {
+                if (runs.ContainsKey(index))
+                {
+                    HandleUpdate(runs[index].Unset(index));
+                }
+                else
+                {
+                    var run = GetRunBefore(index);
+                    if (run == null) return;
+                    HandleUpdate(run.Unset(index));
+                }
+            }
+
+            public void Iterate(Action<int> onIndex)
+            {
+                foreach (var run in runs.Values)
+                {
+                    run.Iterate(onIndex);
+                }
+            }
+
+            public int[] RunLengthEncoded()
+            {
+                return Encode().ToArray();
+            }
+
+            private IEnumerable<int> Encode()
+            {
+                foreach (var pair in runs)
+                {
+                    yield return pair.Value.Start;
+                    yield return pair.Value.Length;
+                }
+            }
+
+            private Run? GetRunBefore(int index)
+            {
+                Run? result = null;
+                foreach (var pair in runs)
+                {
+                    if (pair.Key < index) result = pair.Value;
+                    else return result;
+                }
+                return result;
+            }
+
+            private void HandleUpdate(RunUpdate runUpdate)
+            {
+                foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
+                foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
+            }
+
+            private void CreateNewRun(int index)
+            {
+                if (runs.ContainsKey(index + 1))
+                {
+                    var length = runs[index + 1].Length + 1;
+                    runs.Add(index, new Run(index, length));
+                    runs.Remove(index + 1);
+                }
+                else
+                {
+                    runs.Add(index, new Run(index, 1));
+                }
+            }
+        }
     }
 }

Tools/AutoClient/Purchaser.cs

@@ -68,7 +68,7 @@ namespace AutoClient
                 var filename = Guid.NewGuid().ToString().ToLowerInvariant();
                 {
                     using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
+                    var fileResponse = await codex.DownloadNetworkAsync(cid);
                     fileResponse.Stream.CopyTo(fileStream);
                 }
                 var time = sw.Elapsed;
@@ -84,15 +84,8 @@ namespace AutoClient
         private async Task<string> StartNewPurchase()
         {
             var file = await CreateFile();
-            try
-            {
-                var cid = await UploadFile(file);
-                return await RequestStorage(cid);
-            }
-            finally
-            {
-                DeleteFile(file);
-            }
+            var cid = await UploadFile(file);
+            return await RequestStorage(cid);
         }
 
         private async Task<string> CreateFile()
@@ -100,18 +93,6 @@ namespace AutoClient
             return await app.Generator.Generate();
         }
 
-        private void DeleteFile(string file)
-        {
-            try
-            {
-                File.Delete(file);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error($"Failed to delete file '{file}': {exc}");
-            }
-        }
-
         private async Task<ContentId> UploadFile(string filename)
         {
             using var fileStream = File.OpenRead(filename);

Tools/AutoClient/README.md

@@ -6,7 +6,7 @@ Can generate random images or random data of a specified size.
 
 ## How to run
 
-- dotnet 8.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
+- dotnet 7.0 and CLI arguments: `dotnet run -- --codex-host=... --codex-port=...`
 - docker and env-vars: `codexstorage/codex-autoclient:sha-88daab3`
 
 ## Configuration options

Tools/AutoClient/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/BiblioTech/AdminChecker.cs

@@ -1,7 +1,5 @@
-using BiblioTech.Options;
-using Discord;
+using Discord;
 using Discord.WebSocket;
-using Org.BouncyCastle.Utilities;
 
 namespace BiblioTech
 {
@@ -31,19 +29,7 @@ namespace BiblioTech
 
         public async Task SendInAdminChannel(string msg)
         {
-            await SendInAdminChannel(msg.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
-        }
-
-        public async Task SendInAdminChannel(string[] lines)
-        {
-            var chunker = new LineChunker(lines);
-            var chunks = chunker.GetChunks();
-            if (!chunks.Any()) return;
-
-            foreach (var chunk in chunks)
-            {
-                await adminChannel.SendMessageAsync(string.Join(Environment.NewLine, chunk));
-            }
+            await adminChannel.SendMessageAsync(msg);
         }
 
         public void SetAdminChannel(ISocketMessageChannel adminChannel)

Tools/BiblioTech/BaseCommand.cs

@@ -1,6 +1,7 @@
 using Discord.WebSocket;
 using BiblioTech.Options;
 using Discord;
+using k8s.KubeConfigModels;
 
 namespace BiblioTech
 {

Tools/BiblioTech/BiblioTech.csproj

@@ -12,7 +12,6 @@
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
     <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
-    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 
 </Project>

Tools/BiblioTech/CodexCidChecker.cs

@@ -1,198 +0,0 @@
-using CodexOpenApi;
-using IdentityModel.Client;
-using Utils;
-
-namespace BiblioTech
-{
-    public class CodexCidChecker
-    {
-        private static readonly string nl = Environment.NewLine;
-        private readonly Configuration config;
-        private CodexApi? currentCodexNode;
-
-        public CodexCidChecker(Configuration config)
-        {
-            this.config = config;
-        }
-
-        public async Task<CheckResponse> PerformCheck(string cid)
-        {
-            if (string.IsNullOrEmpty(config.CodexEndpoint))
-            {
-                return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
-            }
-
-            try
-            {
-                var codex = GetCodex();
-                var nodeCheck = await CheckCodex(codex);
-                if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
-
-                return await PerformCheck(codex, cid);
-            }
-            catch (Exception ex)
-            {
-                return new CheckResponse(false, "Internal server error", ex.ToString());
-            }
-        }
-
-        private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
-        {
-            try
-            {
-                var manifest = await codex.DownloadNetworkManifestAsync(cid);
-                return SuccessMessage(manifest);
-            }
-            catch (ApiException apiEx)
-            {
-                if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
-                if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
-                return UnexpectedReturnCode(apiEx.Response);
-            }
-            catch (Exception ex)
-            {
-                return UnexpectedException(ex);
-            }
-        }
-
-        #region Response formatting
-
-        private CheckResponse SuccessMessage(DataItem content)
-        {
-            return FormatResponse(
-                success: true,
-                title: $"Success: '{content.Cid}'",
-                error: "",
-                $"size: {content.Manifest.OriginalBytes} bytes",
-                $"blockSize: {content.Manifest.BlockSize} bytes",
-                $"protected: {content.Manifest.Protected}"
-            );
-        }
-
-        private CheckResponse UnexpectedException(Exception ex)
-        {
-            return FormatResponse(
-                success: false,
-                title: "Unexpected error",
-                error: ex.ToString(),
-                content: "Details will be sent to the bot-admin channel."
-            );
-        }
-
-        private CheckResponse UnexpectedReturnCode(string response)
-        {
-            var msg = "Unexpected return code. Response: " + response;
-            return FormatResponse(
-                success: false,
-                title: "Unexpected return code",
-                error: msg,
-                content: msg
-            );
-        }
-
-        private CheckResponse FailedToFetch(string response)
-        {
-            var msg = "Failed to download content. Response: " + response;
-            return FormatResponse(
-                success: false,
-                title: "Could not download content",
-                error: msg,
-                msg,
-                $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
-            );
-        }
-
-        private CheckResponse CidFormatInvalid(string response)
-        {
-            return FormatResponse(
-                success: false,
-                title: "Invalid format",
-                error: "",
-                content: "Provided CID is not formatted correctly."
-            );
-        }
-
-        private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
-        {
-            var msg = string.Join(nl,
-                new string[]
-                {
-                    title,
-                    "```"
-                }
-                .Concat(content)
-                .Concat(new string[]
-                {
-                    "```"
-                })
-            ) + nl + nl;
-
-            return new CheckResponse(success, msg, error);
-        }
-
-        #endregion
-
-        #region Codex Node API 
-
-        private CodexApi GetCodex()
-        {
-            if (currentCodexNode == null) currentCodexNode = CreateCodex();
-            return currentCodexNode;
-        }
-
-        private async Task<bool> CheckCodex(CodexApi codex)
-        {
-            try
-            {
-                var info = await currentCodexNode!.GetDebugInfoAsync();
-                if (info == null || string.IsNullOrEmpty(info.Id)) return false;
-                return true;
-            }
-            catch (Exception e)
-            {
-                return false;
-            }
-        }
-
-        private CodexApi CreateCodex()
-        {
-            var endpoint = config.CodexEndpoint;
-            var splitIndex = endpoint.LastIndexOf(':');
-            var host = endpoint.Substring(0, splitIndex);
-            var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
-
-            var address = new Address(
-                host: host,
-                port: port
-            );
-
-            var client = new HttpClient();
-            if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
-            {
-                var tokens = config.CodexEndpointAuth.Split(':');
-                if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
-                client.SetBasicAuthentication(tokens[0], tokens[1]);    
-            }
-
-            var codex = new CodexApi(client);
-            codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
-            return codex;
-        }
-
-        #endregion
-    }
-
-    public class CheckResponse
-    {
-        public CheckResponse(bool success, string message, string error)
-        {
-            Success = success;
-            Message = message;
-            Error = error;
-        }
-
-        public bool Success { get; }
-        public string Message { get; }
-        public string Error { get; }
-    }
-}

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -1,38 +0,0 @@
-using BiblioTech.Options;
-
-namespace BiblioTech.Commands
-{
-    public class CheckCidCommand : BaseCommand
-    {
-        private readonly StringOption cidOption = new StringOption(
-            name: "cid",
-            description: "Codex Content-Identifier",
-            isRequired: true);
-        private readonly CodexCidChecker checker;
-
-        public CheckCidCommand(CodexCidChecker checker)
-        {
-            this.checker = checker;
-        }
-
-        public override string Name => "check";
-        public override string StartingMessage => RandomBusyMessage.Get();
-        public override string Description => "Checks if content is available in the testnet.";
-        public override CommandOption[] Options => new[] { cidOption };
-
-        protected override async Task Invoke(CommandContext context)
-        {
-            var user = context.Command.User;
-            var cid = await cidOption.Parse(context);
-            if (string.IsNullOrEmpty(cid))
-            {
-                await context.Followup("Option 'cid' was not received.");
-                return;
-            }
-
-            var response = await checker.PerformCheck(cid);
-            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
-            await context.Followup(response.Message);
-        }
-    }
-}

Tools/BiblioTech/Configuration.cs

@@ -38,12 +38,6 @@ namespace BiblioTech
         [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
         public int NoDiscord { get; set; } = 0;
 
-        [Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")]
-        public string CodexEndpoint { get; set; } = "http://localhost:8080";
-
-        [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
-        public string CodexEndpointAuth { get; set; } = "";
-
         public string EndpointsPath => Path.Combine(DataPath, "endpoints");
         public string UserDataPath => Path.Combine(DataPath, "users");
         public string LogPath => Path.Combine(DataPath, "logs");

Tools/BiblioTech/Program.cs

@@ -3,6 +3,7 @@ using BiblioTech.Commands;
 using BiblioTech.Rewards;
 using Discord;
 using Discord.WebSocket;
+using DiscordRewards;
 using Logging;
 
 namespace BiblioTech
@@ -80,7 +81,6 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checker = new CodexCidChecker(Config);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();
@@ -90,7 +90,6 @@ namespace BiblioTech
                 sprCommand,
                 associateCommand,
                 notifyCommand,
-                new CheckCidCommand(checker),
                 new AdminCommand(sprCommand, replacement)
             );
 

Tools/BiblioTech/Rewards/ChainEventsSender.cs

@@ -1,5 +1,4 @@
 using Discord.WebSocket;
-using DiscordRewards;
 using Logging;
 
 namespace BiblioTech.Rewards
@@ -17,17 +16,24 @@ namespace BiblioTech.Rewards
             this.eventsChannel = eventsChannel;
         }
 
-        public async Task ProcessChainEvents(ChainEventMessage[] eventsOverview, string[] errors)
+        public async Task ProcessChainEvents(string[] eventsOverview)
         {
-            await SendErrorsToAdminChannel(errors);
-
             if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
             try
             {
                 await Task.Run(async () =>
                 {
                     var users = Program.UserRepo.GetAllUserData();
-                    await SendChainEventsInOrder(eventsOverview, eventsChannel, users);
+
+                    foreach (var e in eventsOverview)
+                    {
+                        if (!string.IsNullOrEmpty(e))
+                        {
+                            var @event = ApplyReplacements(users, e);
+                            await eventsChannel.SendMessageAsync(@event);
+                            await Task.Delay(1000);
+                        }
+                    }
                 });
             }
             catch (Exception ex)
@@ -36,37 +42,6 @@ namespace BiblioTech.Rewards
             }
         }
 
-        private async Task SendErrorsToAdminChannel(string[] errors)
-        {
-            try
-            {
-                foreach (var error in errors)
-                {
-                    await Program.AdminChecker.SendInAdminChannel(error);
-                }
-            }
-            catch (Exception exc)
-            {
-                log.Error("Failed to send error messages to admin channel. " + exc);
-                Environment.Exit(1);
-            }
-        }
-
-        private async Task SendChainEventsInOrder(ChainEventMessage[] eventsOverview, SocketTextChannel eventsChannel, UserData[] users)
-        {
-            eventsOverview = eventsOverview.OrderBy(e => e.BlockNumber).ToArray();
-            foreach (var e in eventsOverview)
-            {
-                var msg = e.Message;
-                if (!string.IsNullOrEmpty(msg))
-                {
-                    var @event = ApplyReplacements(users, msg);
-                    await eventsChannel.SendMessageAsync(@event);
-                    await Task.Delay(300);
-                }
-            }
-        }
-
         private string ApplyReplacements(UserData[] users, string msg)
         {
             var result = ApplyUserAddressReplacements(users, msg);

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -31,7 +31,7 @@ namespace BiblioTech.Rewards
                 await ProcessRewards(rewards);
             }
 
-            await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors);
+            await eventsSender.ProcessChainEvents(rewards.EventsOverview);
         }
 
         private async Task ProcessRewards(GiveRewardsCommand rewards)

Tools/BiblioTech/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/CodexNetDeployer/Deployer.cs

@@ -161,7 +161,7 @@ namespace CodexNetDeployer
 
             Log("Starting metrics service...");
 
-            var runningContainer = ci.DeployMetricsCollector(scrapeInterval: TimeSpan.FromSeconds(10.0), startResults.Select(r => r.CodexNode).ToArray());
+            var runningContainer = ci.DeployMetricsCollector(startResults.Select(r => r.CodexNode).ToArray());
 
             Log("Metrics service started.");
 

Tools/CsvCombiner/CsvCombiner.csproj

@@ -1,14 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
-  </ItemGroup>
-
-</Project>

Tools/CsvCombiner/Program.cs

@@ -1,71 +0,0 @@
-using Logging;
-
-public class Program
-{
-    public static void Main(string[] args)
-    {
-        args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"];
-        var p = new Program(args[0]);
-        p.Run();
-    }
-
-    private static readonly ILog log = new ConsoleLog();
-    private string path;
-
-    private readonly Dictionary<string, List<string>> combine = new Dictionary<string, List<string>>();
-
-    public Program(string path)
-    {
-        this.path = path;
-    }
-
-    private void Run()
-    {
-        Log("Starting in " + path);
-
-        var files = Directory.GetFiles(path)
-            .Where(f => f.ToLowerInvariant().EndsWith(".csv")).ToArray();
-
-        foreach (var file in files)
-        {
-            AddToMap(file);
-        }
-
-        var i = 0;
-        foreach (var pair in combine)
-        {
-            var list = pair.Value;
-            list.Insert(0, pair.Key);
-
-            File.WriteAllLines(Path.Combine(path, "combine_" + i + ".csv"), list.ToArray());
-            i++;
-        }
-
-        Log("done");
-    }
-
-    private void AddToMap(string file)
-    {
-        var lines = File.ReadAllLines(file);
-        if (lines.Length > 1)
-        {
-            var header = lines[0];
-            var list = GetList(header);
-            list.AddRange(lines.Skip(1));
-        }
-    }
-
-    private List<string> GetList(string header)
-    {
-        if (!combine.ContainsKey(header))
-        {
-            combine.Add(header, new List<string>());
-        }
-        return combine[header];
-    }
-
-    private void Log(string msg)
-    {
-        log.Log(msg);
-    }
-}

Tools/KeyMaker/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/MarketInsights/AverageHistory.cs

@@ -1,5 +1,6 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
+using Nethereum.Model;
 using TestNetRewarder;
 using Utils;
 
@@ -39,7 +40,7 @@ namespace MarketInsights
 
         private MarketTimeSegment BuildContribution(TimeRange timeRange)
         {
-            var builder = new ContributionBuilder(appState.Log, timeRange);
+            var builder = new ContributionBuilder(timeRange);
             mux.Handlers.Add(builder);
             chainState.Update(timeRange.To);
             mux.Handlers.Remove(builder);

Tools/MarketInsights/ContributionBuilder.cs

@@ -1,6 +1,5 @@
 using CodexContractsPlugin.ChainMonitor;
 using GethPlugin;
-using Logging;
 using System.Numerics;
 using Utils;
 
@@ -9,16 +8,14 @@ namespace MarketInsights
     public class ContributionBuilder : IChainStateChangeHandler
     {
         private readonly MarketTimeSegment segment = new MarketTimeSegment();
-        private readonly ILog log;
 
-        public ContributionBuilder(ILog log, TimeRange timeRange)
+        public ContributionBuilder(TimeRange timeRange)
         {
             segment = new MarketTimeSegment
             {
                 FromUtc = timeRange.From,
                 ToUtc = timeRange.To
             };
-            this.log = log;
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
@@ -54,15 +51,6 @@ namespace MarketInsights
         {
         }
 
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-        }
-
-        public void OnError(string msg)
-        {
-            log.Error(msg);
-        }
-
         public MarketTimeSegment GetSegment()
         {
             return segment;

Tools/TestNetRewarder/BotClient.cs

@@ -18,6 +18,7 @@ namespace TestNetRewarder
         public async Task<bool> IsOnline()
         {
             var result = await HttpGet();
+            log.Log("Is DiscordBot online: " + result);
             return result == "Pong";
         }
 

Tools/TestNetRewarder/EmojiMaps.cs

@@ -1,83 +0,0 @@
-using Utils;
-
-namespace TestNetRewarder
-{
-    public class EmojiMaps
-    {
-        private readonly string[] emojis = new[]
-        {
-            // yellow
-            "😀",
-            "🌻",
-            "🍋",
-            "🧀",
-            "🌔",
-            "⭐",
-            "⚡",
-            "🏆",
-            // red
-            "💘",
-            "🦞",
-            "🌹",
-            "🍒",
-            "🫖", // teapot
-            "⛩",
-            "🚗",
-            "🔥",
-            // green
-            "🐊",
-            "🦎",
-            "🐛",
-            "🌳",
-            "🍀",
-            "🧩",
-            "🔋",
-            "♻",
-            // blue
-            "💙",
-            "🐳",
-            "🐟",
-            "🍉",
-            "🧊",
-            "🌐",
-            "⚓",
-            "🌀",
-        };
-
-        public string NewRequest => "🐟";
-        public string Started => "🦈";
-        public string SlotFilled => "🟢";
-        public string SlotFreed => "⭕";
-        public string SlotReservationsFull => "☑️";
-        public string Finished => "✅";
-        public string Cancelled => "🚫";
-        public string Failed => "❌";
-
-        public string StringToEmojis(string input, int outLength)
-        {
-            if (outLength < 1) outLength = 1;
-
-            var result = "";
-            var segmentLength = input.Length / outLength;
-            if (segmentLength < 1)
-            {
-                return StringToEmojis(input + input, outLength);
-            }
-            for (var i = 0; i < outLength; i++)
-            {
-                var segment = input.Substring(i * segmentLength, segmentLength);
-                result += SelectOne(segment);
-            }
-
-            return result;
-        }
-
-        private string SelectOne(string segment)
-        {
-            var index = 0;
-            foreach (var c in segment) index += Convert.ToInt32(c);
-            index = index % emojis.Length;
-            return emojis[index];
-        }
-    }
-}

Tools/TestNetRewarder/EventsFormatter.cs

@@ -1,6 +1,5 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
-using DiscordRewards;
 using GethPlugin;
 using System.Globalization;
 using System.Numerics;
@@ -11,28 +10,24 @@ namespace TestNetRewarder
     public class EventsFormatter : IChainStateChangeHandler
     {
         private static readonly string nl = Environment.NewLine;
-        private readonly List<ChainEventMessage> events = new List<ChainEventMessage>();
-        private readonly List<string> errors = new List<string>();
-        private readonly EmojiMaps emojiMaps = new EmojiMaps();
+        private readonly List<string> events = new List<string>();
 
-        public ChainEventMessage[] GetEvents()
+        public string[] GetEvents()
         {
             var result = events.ToArray();
             events.Clear();
             return result;
         }
 
-        public string[] GetErrors()
+        public void AddError(string error)
         {
-            var result = errors.ToArray();
-            errors.Clear();
-            return result;
+            AddBlock("📢 **Error**", error);
         }
 
         public void OnNewRequest(RequestEvent requestEvent)
         {
             var request = requestEvent.Request;
-            AddRequestBlock(requestEvent, $"{emojiMaps.NewRequest} New Request",
+            AddRequestBlock(requestEvent, "New Request",
                 $"Client: {request.Client}",
                 $"Content: {request.Request.Content.Cid}",
                 $"Duration: {BigIntToDuration(request.Request.Ask.Duration)}",
@@ -47,27 +42,27 @@ namespace TestNetRewarder
 
         public void OnRequestCancelled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Cancelled} Cancelled");
+            AddRequestBlock(requestEvent, "Cancelled");
         }
 
         public void OnRequestFailed(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Failed} Failed");
+            AddRequestBlock(requestEvent, "Failed");
         }
 
         public void OnRequestFinished(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Finished} Finished");
+            AddRequestBlock(requestEvent, "Finished");
         }
 
         public void OnRequestFulfilled(RequestEvent requestEvent)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.Started} Started");
+            AddRequestBlock(requestEvent, "Started");
         }
 
         public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFilled} Slot Filled",
+            AddRequestBlock(requestEvent, "Slot Filled",
                 $"Host: {host}",
                 $"Slot Index: {slotIndex}"
             );
@@ -75,46 +70,24 @@ namespace TestNetRewarder
 
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotFreed} Slot Freed",
+            AddRequestBlock(requestEvent, "Slot Freed",
                 $"Slot Index: {slotIndex}"
             );
         }
 
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            AddRequestBlock(requestEvent, $"{emojiMaps.SlotReservationsFull} Slot Reservations Full",
-                $"Slot Index: {slotIndex}"
-            );
-        }
-        
-        public void OnError(string msg)
-        {
-            errors.Add(msg);
-        }
-
         private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
         {
             var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
-            var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}";
-            AddBlock(requestEvent.Block.BlockNumber, title, content);
+            var title = $"{blockNumber} **{eventName}** `{requestEvent.Request.Request.Id}`";
+            AddBlock(title, content);
         }
 
-        private void AddBlock(ulong blockNumber, string title, params string[] content)
+        private void AddBlock(string title, params string[] content)
         {
-            events.Add(FormatBlock(blockNumber, title, content));
+            events.Add(FormatBlock(title, content));
         }
 
-        private ChainEventMessage FormatBlock(ulong blockNumber, string title, params string[] content)
-        {
-            var msg = FormatBlockMessage(title, content);
-            return new ChainEventMessage
-            {
-                BlockNumber = blockNumber,
-                Message = msg
-            };
-        }
-
-        private string FormatBlockMessage(string title, string[] content)
+        private string FormatBlock(string title, params string[] content)
         {
             if (content == null || !content.Any())
             {
@@ -140,13 +113,6 @@ namespace TestNetRewarder
             return utc.ToString("yyyy-MM-dd HH:mm:ss UTC", CultureInfo.InvariantCulture);
         }
 
-        private string FormatRequestId(RequestEvent requestEvent)
-        {
-            return
-                $"({emojiMaps.StringToEmojis(requestEvent.Request.Request.Id, 3)})" +
-                $"`{requestEvent.Request.Request.Id}`";
-        }
-
         private string BigIntToDuration(BigInteger big)
         {
             var span = TimeSpan.FromSeconds((int)big);

Tools/TestNetRewarder/Processor.cs

@@ -48,7 +48,7 @@ namespace TestNetRewarder
             {
                 var msg = "Exception processing time segment: " + ex;
                 log.Error(msg); 
-                eventsFormatter.OnError(msg);
+                eventsFormatter.AddError(msg);
                 throw;
             }
         }
@@ -58,9 +58,8 @@ namespace TestNetRewarder
             var numberOfChainEvents = chainState.Update(timeRange.To);
 
             var events = eventsFormatter.GetEvents();
-            var errors = eventsFormatter.GetErrors();
 
-            var request = builder.Build(events, errors);
+            var request = builder.Build(events);
             if (request.HasAny())
             {
                 await client.SendRewards(request);

Tools/TestNetRewarder/RequestBuilder.cs

@@ -19,7 +19,7 @@ namespace TestNetRewarder
             }
         }
 
-        public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors)
+        public GiveRewardsCommand Build(string[] lines)
         {
             var result = new GiveRewardsCommand
             {
@@ -28,8 +28,7 @@ namespace TestNetRewarder
                     RewardId = p.Key,
                     UserAddresses = p.Value.Select(v => v.Address).ToArray()
                 }).ToArray(),
-                EventsOverview = lines,
-                Errors = errors
+                EventsOverview = lines
             };
 
             rewards.Clear();

Tools/TestNetRewarder/RewardCheck.cs

@@ -72,14 +72,6 @@ namespace TestNetRewarder
         {
         }
 
-        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-        }
-
-        public void OnError(string msg)
-        {
-        }
-
         private void GiveReward(RewardConfig reward, EthAddress receiver)
         {
             giver.Give(reward, receiver);

Tools/TestNetRewarder/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Variables
-ARG BUILDER=mcr.microsoft.com/dotnet/sdk:8.0
+ARG BUILDER=mcr.microsoft.com/dotnet/sdk:7.0
 ARG IMAGE=${BUILDER}
 ARG APP_HOME=/app
 

Tools/TranscriptAnalysis/CsvWriter.cs

@@ -1,136 +0,0 @@
-using Logging;
-
-namespace TranscriptAnalysis
-{
-    public class CsvWriter
-    {
-        private readonly ILog log;
-
-        public CsvWriter(ILog log)
-        {
-            this.log = log;
-        }
-
-        public ICsv CreateNew()
-        {
-            return new Csv();
-        }
-
-        public void Write(ICsv csv, string filename)
-        {
-            var c = (Csv)csv;
-
-            using var file = File.OpenWrite(filename);
-            using var writer = new StreamWriter(file);
-            c.CreateLines(writer.WriteLine);
-
-            log.Log($"CSV written to: '{filename}'");
-        }
-    }
-
-    public interface ICsv
-    {
-        ICsvColumn GetColumn(string title, float defaultValue);
-        ICsvColumn GetColumn(string title, string defaultValue);
-        void AddRow(params CsvCell[] cells);
-    }
-
-    public class Csv : ICsv
-    {
-        private readonly string Sep = ",";
-        private readonly List<CsvColumn> columns = new List<CsvColumn>();
-        private readonly List<CsvRow> rows = new List<CsvRow>();
-
-        public ICsvColumn GetColumn(string title, float defaultValue)
-        {
-            return GetColumn(title, defaultValue.ToString());
-        }
-
-        public ICsvColumn GetColumn(string title, string defaultValue)
-        {
-            var column = columns.SingleOrDefault(c => c.Title == title);
-            if (column == null)
-            {
-                column = new CsvColumn(title, defaultValue);
-                columns.Add(column);
-            }
-            return column;
-        }
-
-        public void AddRow(params CsvCell[] cells)
-        {
-            rows.Add(new CsvRow(cells));
-        }
-
-        public void CreateLines(Action<string> onLine)
-        {
-            CreateHeaderLine(onLine);
-            foreach (var row in rows)
-            {
-                CreateRowLine(row, onLine);
-            }
-        }
-
-        private void CreateHeaderLine(Action<string> onLine)
-        {
-            onLine(string.Join(Sep, columns.Select(c => c.Title).ToArray()));
-        }
-
-        private void CreateRowLine(CsvRow row, Action<string> onLine)
-        {
-            onLine(string.Join(Sep, columns.Select(c => GetRowCellValue(row, c)).ToArray()));
-        }
-
-        private string GetRowCellValue(CsvRow row, CsvColumn column)
-        {
-            var cell = row.Cells.SingleOrDefault(c => c.Column == column);
-            if (cell == null) return column.DefaultValue;
-            return cell.Value;
-        }
-    }
-
-    public class CsvCell
-    {
-        public CsvCell(ICsvColumn column, float value)
-            : this(column, value.ToString())
-        {
-        }
-
-        public CsvCell(ICsvColumn column, string value)
-        {
-            Column = column;
-            Value = value;
-        }
-
-        public ICsvColumn Column { get; }
-        public string Value { get; }
-    }
-
-    public interface ICsvColumn
-    {
-        string Title { get; }
-        string DefaultValue { get; }
-    }
-
-    public class CsvColumn : ICsvColumn
-    {
-        public CsvColumn(string title, string defaultValue)
-        {
-            Title = title;
-            DefaultValue = defaultValue;
-        }
-
-        public string Title { get; }
-        public string DefaultValue { get; }
-    }
-
-    public class CsvRow
-    {
-        public CsvRow(CsvCell[] cells)
-        {
-            Cells = cells;
-        }
-
-        public CsvCell[] Cells { get; }
-    }
-}

Tools/TranscriptAnalysis/Program.cs

@@ -9,6 +9,8 @@ public static class Program
 
     public static void Main(string[] args)
     {
+        //args = new[] { "D:\\Projects\\cs-codex-dist-tests\\Tests\\CodexTests\\bin\\Debug\\net7.0\\CodexTestLogs\\2024-08\\06\\08-24-45Z_ThreeClientTest\\SwarmTest_SwarmTest.owts" };
+
         Log("Transcript Analysis");
         if (!args.Any())
         {
@@ -31,7 +33,7 @@ public static class Program
         };
 
         var header = reader.GetHeader<OverwatchCodexHeader>("cdx_h");
-        var receivers = new ReceiverSet(args[0], log, reader, header);
+        var receivers = new ReceiverSet(log, reader, header);
         receivers.InitAll();
 
         var processor = new Processor(log, reader);

Tools/TranscriptAnalysis/ReceiverSet.cs

@@ -7,7 +7,7 @@ namespace TranscriptAnalysis
 {
     public interface IEventReceiver
     {
-        void Init(string sourceFilename, ILog log, OverwatchCodexHeader header);
+        void Init(ILog log, OverwatchCodexHeader header);
         void Finish();
     }
 
@@ -18,15 +18,13 @@ namespace TranscriptAnalysis
 
     public class ReceiverSet
     {
-        private readonly string sourceFilename;
         private readonly ILog log;
         private readonly ITranscriptReader reader;
         private readonly OverwatchCodexHeader header;
         private readonly List<IEventReceiver> receivers = new List<IEventReceiver>();
 
-        public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
+        public ReceiverSet(ILog log, ITranscriptReader reader, OverwatchCodexHeader header)
         {
-            this.sourceFilename = sourceFilename;
             this.log = log;
             this.reader = reader;
             this.header = header;
@@ -55,7 +53,7 @@ namespace TranscriptAnalysis
             mux.Add(receiver);
 
             receivers.Add(receiver);
-            receiver.Init(sourceFilename, log, header);
+            receiver.Init(log, header);
         }
 
         // We use a mux here because, for each time we call reader.AddEventHandler,

Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs

@@ -8,38 +8,29 @@ namespace TranscriptAnalysis.Receivers
     {
         protected ILog log { get; private set; } = new NullLog();
         protected OverwatchCodexHeader Header { get; private set; } = null!;
-        protected CsvWriter CsvWriter { get; private set; }
-        protected string SourceFilename { get; private set; } = string.Empty;
 
         public abstract string Name { get; }
         public abstract void Receive(ActivateEvent<T> @event);
         public abstract void Finish();
 
-        protected BaseReceiver()
-        {
-            CsvWriter = new CsvWriter(log);
-        }
-
-        public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header)
+        public void Init(ILog log, OverwatchCodexHeader header)
         {
             this.log = new LogPrefixer(log, $"({Name}) ");
             Header = header;
-            SourceFilename = sourceFilename;
         }
 
-        protected string? GetPeerId(int nodeIndex)
+        protected string GetPeerId(int nodeIndex)
         {
-            return GetIdentity(nodeIndex)?.PeerId;
+            return GetIdentity(nodeIndex).PeerId;
         }
 
-        protected string? GetName(int nodeIndex)
+        protected string GetName(int nodeIndex)
         {
-            return GetIdentity(nodeIndex)?.Name;
+            return GetIdentity(nodeIndex).Name;
         }
 
-        protected CodexNodeIdentity? GetIdentity(int nodeIndex)
+        protected CodexNodeIdentity GetIdentity(int nodeIndex)
         {
-            if (nodeIndex < 0) return null;
             return Header.Nodes[nodeIndex];
         }
 

Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs

@@ -5,9 +5,6 @@ namespace TranscriptAnalysis.Receivers
 {
     public class DuplicateBlocksReceived : BaseReceiver<OverwatchCodexEvent>
     {
-        public static List<int> Counts = new List<int>();
-        private long uploadSize;
-
         public override string Name => "BlocksReceived";
 
         public override void Receive(ActivateEvent<OverwatchCodexEvent> @event)
@@ -16,17 +13,11 @@ namespace TranscriptAnalysis.Receivers
             {
                 Handle(@event.Payload, @event.Payload.BlockReceived);
             }
-            if (@event.Payload.FileUploaded != null)
-            {
-                var uploadEvent = @event.Payload.FileUploaded;
-                uploadSize = uploadEvent.ByteSize;
-            }
         }
 
         public override void Finish()
         {
             Log("Number of BlockReceived events seen: " + seen);
-            var csv = CsvWriter.CreateNew();
 
             var totalReceived = peerIdBlockAddrCount.Sum(a => a.Value.Sum(p => p.Value));
             var maxRepeats = peerIdBlockAddrCount.Max(a => a.Value.Max(p => p.Value));
@@ -40,26 +31,13 @@ namespace TranscriptAnalysis.Receivers
                 }
             }
 
-            if (Counts.Any()) throw new Exception("Should be empty");
-
             float t = totalReceived;
-            csv.GetColumn("numNodes", Header.Nodes.Length);
-            csv.GetColumn("filesize", uploadSize.ToString());
-            var receiveCountColumn = csv.GetColumn("receiveCount", 0.0f);
-            var occuranceColumn = csv.GetColumn("occurance", 0.0f);
             occurances.PrintContinous((i, count) =>
             {
                 float n = count;
                 float p = 100.0f * (n / t);
                 Log($"Block received {i} times = {count}x ({p}%)");
-                Counts.Add(count);
-                csv.AddRow(
-                    new CsvCell(receiveCountColumn, i),
-                    new CsvCell(occuranceColumn, count)
-                );
             });
-
-            CsvWriter.Write(csv, SourceFilename + "_blockduplicates.csv");
         }
 
         private int seen = 0;
@@ -68,7 +46,6 @@ namespace TranscriptAnalysis.Receivers
         private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived)
         {
             var receiverPeerId = GetPeerId(payload.NodeIdentity);
-            if (receiverPeerId == null) return;
             var blockAddress = blockReceived.BlockAddress;
             seen++;
 

Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs

@@ -14,8 +14,6 @@ namespace TranscriptAnalysis.Receivers
         {
             var peerId = GetPeerId(@event.Payload.NodeIdentity);
             var name = GetName(@event.Payload.NodeIdentity);
-            if (peerId == null) return;
-            if (name == null) return;
 
             if (!seen.Contains(peerId))
             {

Tools/TranscriptAnalysis/Receivers/NodesDegree.cs

@@ -46,7 +46,6 @@ namespace TranscriptAnalysis.Receivers
 
         private readonly Dictionary<string, Node> dialingNodes = new Dictionary<string, Node>();
         private readonly Dictionary<string, Dial> dials = new Dictionary<string, Dial>();
-        private long uploadSize;
 
         public override string Name => "NodesDegree";
 
@@ -55,20 +54,12 @@ namespace TranscriptAnalysis.Receivers
             if (@event.Payload.DialSuccessful != null)
             {
                 var peerId = GetPeerId(@event.Payload.NodeIdentity);
-                if (peerId == null) return;
                 AddDial(peerId, @event.Payload.DialSuccessful.TargetPeerId);
             }
-            if (@event.Payload.FileUploaded != null)
-            {
-                var uploadEvent = @event.Payload.FileUploaded;
-                uploadSize = uploadEvent.ByteSize;
-            }
         }
 
         public override void Finish()
         {
-            var csv = CsvWriter.CreateNew();
-
             var numNodes = dialingNodes.Count;
             var redialOccurances = new OccuranceMap();
             foreach (var dial in dials.Values)
@@ -89,22 +80,12 @@ namespace TranscriptAnalysis.Receivers
             });
 
             float tot = numNodes;
-            csv.GetColumn("numNodes", Header.Nodes.Length);
-            csv.GetColumn("filesize", uploadSize.ToString());
-            var degreeColumn = csv.GetColumn("degree", 0.0f);
-            var occuranceColumn = csv.GetColumn("occurance", 0.0f);
             degreeOccurances.Print((i, count) =>
             {
                 float n = count;
                 float p = 100.0f * (n / tot);
                 Log($"Degree: {i} = {count}x ({p}%)");
-                csv.AddRow(
-                    new CsvCell(degreeColumn, i),
-                    new CsvCell(occuranceColumn, n)
-                );
             });
-
-            CsvWriter.Write(csv, SourceFilename + "_nodeDegrees.csv");
         }
 
         private void AddDial(string peerId, string targetPeerId)

cs-codex-dist-testing.sln

@@ -76,8 +76,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -204,10 +202,6 @@ Global
 		{004614DF-1C65-45E3-882D-59AE44282573}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{004614DF-1C65-45E3-882D-59AE44282573}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6230347F-5045-4E25-8E7A-13D7221B7444}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6230347F-5045-4E25-8E7A-13D7221B7444}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -243,7 +237,6 @@ Global
 		{870DDFBE-D7ED-4196-9681-13CA947BDEA6} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
 		{C0EEBD32-23CB-45EC-A863-79FB948508C8} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{004614DF-1C65-45E3-882D-59AE44282573} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
-		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:8.0
+FROM mcr.microsoft.com/dotnet/sdk:7.0
 
 COPY --chmod=0755 docker/docker-entrypoint.sh /
 

docker/deployandrun.Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:8.0
+FROM mcr.microsoft.com/dotnet/sdk:7.0
 
 RUN apt-get update && apt-get install -y screen
 WORKDIR /app

docs/Automation.md

@@ -104,7 +104,7 @@ GitHub --> CI --> Kubernetes -->  Job                  |
     spec:
       containers:
       - name: dotnet
-        image: mcr.microsoft.com/dotnet/sdk:8.0
+        image: mcr.microsoft.com/dotnet/sdk:7.0
         env:
         - name: RUNNERLOCATION
           value: InternalToCluster

docs/Continuous-Tests.md

@@ -44,7 +44,7 @@
     spec:
       containers:
       - name: runner
-        image: mcr.microsoft.com/dotnet/sdk:8.0
+        image: mcr.microsoft.com/dotnet/sdk:7.0
         env:
         - name: KUBECONFIG
           value: /opt/kubeconfig.yaml

docs/LOCALSETUP.md

@@ -4,7 +4,7 @@
 These steps will help you set up everything you need to run and debug the tests on your local system.
 
 ### Installing the requirements.
-1. Install dotnet v8.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net8.0` with your version.)
+1. Install dotnet v7.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net7.0` with your version.)
 1. Set up a nice C# IDE or plugin for your current IDE.
 1. Install docker desktop.
 1. In the docker-desktop settings, enable kubernetes. (This might take a few minutes.)