Merge branch 'master' into findbug/networkpeer-crash

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/Configuration.cs

@@ -21,5 +21,8 @@ namespace KubernetesWorkflow
 
         [JsonIgnore]
         public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
+
+        [JsonIgnore]
+        public Func<string?, string?> Replacer { get; set; } = s => s;
     }
 }

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,11 +11,13 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
+        private readonly Func<string?, string?> replacer;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace,
+            Func<string?, string?> replacer)
         {
             this.log = log;
             this.config = config;
@@ -23,6 +25,7 @@ namespace KubernetesWorkflow
             this.podName = podName;
             this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
+            this.replacer = replacer;
             cts = new CancellationTokenSource();
         }
 
@@ -92,7 +95,7 @@ namespace KubernetesWorkflow
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
             var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
-            handler.Log(stream);
+            handler.Log(stream, replacer);
         }
     }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -12,10 +12,11 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly K8sCluster cluster;
         private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly Func<string?, string?> replacer;
         private readonly K8sClient client;
         public const string PodLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace, Func<string?, string?> replacer)
         {
             this.log = log;
             this.cluster = cluster;
@@ -23,6 +24,7 @@ namespace KubernetesWorkflow
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sNamespace = k8sNamespace;
+            this.replacer = replacer;
         }
 
         public void Dispose()
@@ -64,7 +66,7 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous, Func<string?, string?> replacer)
         {
             log.Debug();
 
@@ -72,7 +74,7 @@ namespace KubernetesWorkflow
             var recipeName = container.Recipe.Name;
 
             using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
-            logHandler.Log(stream);
+            logHandler.Log(stream, replacer);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -906,7 +908,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true, replacer);
 
                 throw new Exception(msg);
             }
@@ -952,7 +954,7 @@ namespace KubernetesWorkflow
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace, replacer);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -4,19 +4,19 @@ namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
-        void Log(Stream log);
+        void Log(Stream log, Func<string?, string?> replacer);
     }
 
     public abstract class LogHandler : ILogHandler
     {
-        public void Log(Stream log)
+        public void Log(Stream log, Func<string?, string?> replacer)
         {
             using var reader = new StreamReader(log);
             var line = reader.ReadLine();
             while (line != null)
             {
-                ProcessLine(line);
-                line = reader.ReadLine();
+                line = replacer(reader.ReadLine());
+                if (line != null) ProcessLine(line);
             }
         }
 

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -28,16 +28,17 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly string k8sNamespace;
+        private readonly Func<string?, string?> replacer;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace, Func<string?, string?> replacer)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.k8sNamespace = k8sNamespace;
-
+            this.replacer = replacer;
             locationProvider = new LocationProvider(log, K8s);
         }
 
@@ -119,7 +120,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
             });
         }
 
@@ -131,7 +132,7 @@ namespace KubernetesWorkflow
 
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
             });
 
             return new DownloadedLog(logHandler, container.Name);
@@ -257,7 +258,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
                 action(controller);
                 controller.Dispose();
             }
@@ -272,7 +273,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
                 var result = action(controller);
                 controller.Dispose();
                 return result;

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -24,13 +24,12 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride), configuration.Replacer);
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Utils/RandomUtils.cs

@@ -3,24 +3,31 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
         }
 
         public static T[] Shuffled<T>(T[] items)
         {
-            var result = new List<T>();
-            var source = items.ToList();
-            while (source.Any())
+            lock (@lock)
             {
-                result.Add(RandomUtils.PickOneRandom(source));
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
             }
-            return result.ToArray();
         }
     }
 }

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -48,18 +48,19 @@ namespace CodexContractsPlugin.ChainMonitor
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public void Update()
+        public int Update()
         {
-            Update(DateTime.UtcNow);
+            return Update(DateTime.UtcNow);
         }
 
-        public void Update(DateTime toUtc)
+        public int Update(DateTime toUtc)
         {
             var span = new TimeRange(TotalSpan.To, toUtc);
             var events = ChainEvents.FromTimeRange(contracts, span);
             Apply(events);
 
             TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
         }
 
         private void Apply(ChainEvents events)

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-fb25372";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "6B-94-24-A4-D5-01-6F-12-E9-34-74-36-80-57-7A-3A-79-8C-E8-02-68-B7-05-DA-50-A0-5C-B1-02-B9-AE-C6";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -92,6 +92,12 @@ namespace CodexPlugin
             return mapper.Map(read);
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
@@ -189,7 +195,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,8 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
-
+        private const string DefaultDockerImage = "thatbenbierens/nim-codex:netpeerdebug6";
+            //"codexstorage/nim-codex:0.1.4";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -244,6 +244,11 @@ namespace CodexPlugin
             Version = debugInfo.Version;
         }
 
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
             // The peer we want to connect is in a different pod.

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -63,6 +63,26 @@ namespace CodexPlugin
             };
         }
 
+        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        {
+            return availabilities.Select(a => Map(a)).ToArray();
+        }
+
+        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        {
+            return new StorageAvailability
+            (
+                ToByteSize(availability.TotalSize),
+                ToTimespan(availability.Duration),
+                new TestToken(ToBigIng(availability.MinPrice)),
+                new TestToken(ToBigIng(availability.MaxCollateral))
+            )
+            {
+                Id = availability.Id,
+                FreeSpace = ToByteSize(availability.FreeSize),
+            };
+        }
+
         // TODO: Fix openapi spec for this call.
         //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
         //{
@@ -105,19 +125,6 @@ namespace CodexPlugin
         //    };
         //}
 
-        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
-        {
-            return new StorageAvailability(
-                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
-                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
-                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
-            )
-            {
-                Id = read.Id
-            };
-        }
-
         public CodexSpace Map(Space space)
         {
             return new CodexSpace
@@ -222,5 +229,20 @@ namespace CodexPlugin
         {
             return t.TstWei.ToString("D");
         }
+
+        private BigInteger ToBigIng(string tokens)
+        {
+            return BigInteger.Parse(tokens);
+        }
+
+        private TimeSpan ToTimespan(string duration)
+        {
+            return TimeSpan.FromSeconds(Convert.ToInt32(duration));
+        }
+
+        private ByteSize ToByteSize(string size)
+        {
+            return new ByteSize(Convert.ToInt64(size));
+        }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -7,6 +7,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
+        StorageAvailability[] GetAvailabilities();
         IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
@@ -61,6 +62,14 @@ namespace CodexPlugin
             return response.Id;
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var result = codexAccess.GetAvailabilities();
+            Log($"Got {result.Length} availabilities:");
+            foreach (var a in result) a.Log(log);
+            return result;
+        }
+
         private void Log(string msg)
         {
             log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
@@ -81,6 +90,12 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         private void Unavailable()
         {
             FrameworkAssert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -84,10 +84,11 @@ namespace CodexPlugin
         public TimeSpan MaxDuration { get; }
         public TestToken MinPriceForTotalSpace { get; }
         public TestToken MaxCollateral { get; } 
+        public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
         {
-            log.Log($"Making storage available... (" +
+            log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
                 $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -23,6 +23,8 @@ components:
     Id:
       type: string
       description: 32bits identifier encoded in hex-decimal string.
+      minLength: 66
+      maxLength: 66
       example: 0x...
 
     BigInt:
@@ -136,7 +138,7 @@ components:
           $ref: "#/components/schemas/Duration"
         minPrice:
           type: string
-          description: Minimum price to be paid (in amount of tokens) as decimal string
+          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
         maxCollateral:
           type: string
           description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@@ -168,7 +170,39 @@ components:
           $ref: "#/components/schemas/StorageRequest"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
+
+    SlotAgent:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        slotIndex:
+          type: string
+          description: Slot Index as decimal string
+        requestId:
+          $ref: "#/components/schemas/Id"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        reservation:
+          $ref: "#/components/schemas/Reservation"
+        state:
+          type: string
+          description: Description of the slot's
+          enum:
+            - SaleCancelled
+            - SaleDownloading
+            - SaleErrored
+            - SaleFailed
+            - SaleFilled
+            - SaleFilling
+            - SaleFinished
+            - SaleIgnored
+            - SaleInitialProving
+            - SalePayout
+            - SalePreparing
+            - SaleProving
+            - SaleUnknown
 
     Reservation:
       type: object
@@ -183,7 +217,7 @@ components:
           $ref: "#/components/schemas/Id"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
 
     StorageRequestCreation:
       type: object
@@ -259,6 +293,15 @@ components:
         state:
           type: string
           description: Description of the Request's state
+          enum:
+            - cancelled
+            - error
+            - failed
+            - finished
+            - pending
+            - started
+            - submitted
+            - unknown
         error:
           type: string
           description: If Request failed, then here is presented the error message
@@ -491,7 +534,7 @@ paths:
                   $ref: "#/components/schemas/Slot"
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/slots/{slotId}":
     get:
@@ -511,7 +554,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Slot"
+                $ref: "#/components/schemas/SlotAgent"
 
         "400":
           description: Invalid or missing SlotId
@@ -520,13 +563,13 @@ paths:
           description: Host is not in an active sale for the slot
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
       tags: [ Marketplace ]
-      operationId: getOfferedStorage
+      operationId: getAvailabilities
       responses:
         "200":
           description: Retrieved storage availabilities of the node
@@ -535,11 +578,11 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/SalesAvailability"
+                  $ref: "#/components/schemas/SalesAvailabilityREAD"
         "500":
           description: Error getting unused availabilities
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
     post:
       summary: "Offers storage for sale"
@@ -564,7 +607,7 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
   "/sales/availability/{id}":
     patch:
       summary: "Updates availability"
@@ -597,10 +640,10 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability/{id}/reservations":
-    patch:
+    get:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
@@ -628,7 +671,7 @@ paths:
         "500":
           description: Error getting reservations
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/storage/request/{cid}":
     post:
@@ -659,7 +702,7 @@ paths:
         "404":
           description: Request ID not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases":
     get:
@@ -676,7 +719,7 @@ paths:
                 items:
                   type: string
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases/{id}":
     get:
@@ -702,7 +745,7 @@ paths:
         "404":
           description: Purchase not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/node/spr":
     get:

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/GethPlugin/EthTokenExtensions.cs

@@ -28,7 +28,13 @@
 
         public override string ToString()
         {
-            return $"{Eth} Eth";
+            var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
+
+            var tokens = new List<string>();
+            if (Eth > 0) tokens.Add($"{Eth} Eth");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
+
+            return string.Join(" + ", tokens);
         }
     }
 

ProjectPlugins/GethPlugin/GethNode.cs

@@ -57,7 +57,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/GethPlugin/GethPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/MetricsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -80,7 +80,7 @@ namespace MetricsPlugin
     {
         public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -1,28 +1,46 @@
 using CodexPlugin;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace CodexTests
 {
     public class AutoBootstrapDistTest : CodexDistTest
     {
+        private readonly Dictionary<TestLifecycle, ICodexNode> bootstrapNodes = new Dictionary<TestLifecycle, ICodexNode>();
+
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
+            var tl = Get();
+            if (!bootstrapNodes.ContainsKey(tl))
+            {
+                bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace)));
+            }
         }
 
         [TearDown]
         public void TearDownBootstrapNode()
         {
-            BootstrapNode = null;
+            bootstrapNodes.Remove(Get());
         }
 
         protected override void OnCodexSetup(ICodexSetup setup)
         {
-            if (BootstrapNode != null) setup.WithBootstrapNode(BootstrapNode);
+            var node = BootstrapNode;
+            if (node != null) setup.WithBootstrapNode(node);
         }
 
-        protected ICodexNode? BootstrapNode { get; private set; }
-        
+        protected ICodexNode? BootstrapNode
+        {
+            get
+            {
+                var tl = Get();
+                if (bootstrapNodes.TryGetValue(tl, out var node))
+                {
+                    return node;
+                }
+                return null;
+            }
+        }
     }
 }

Tests/CodexTests/BasicTests/EventLogginHandler.cs

@@ -0,0 +1,57 @@
+using CodexContractsPlugin.ChainMonitor;
+using GethPlugin;
+using Logging;
+using System.Numerics;
+
+namespace CodexTests.BasicTests
+{
+    public class EventLogginHandler : IChainStateChangeHandler
+    {
+        private readonly ILog log;
+
+        public EventLogginHandler(ILog log)
+        {
+            this.log = log;
+        }
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            Log(nameof(OnNewRequest), requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestCancelled), requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFailed), requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFinished), requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            Log(nameof(OnRequestFulfilled), requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            Log(nameof(OnSlotFilled), requestEvent, host.ToString(), slotIndex.ToString());
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            Log(nameof(OnNewRequest), requestEvent, slotIndex.ToString());
+        }
+
+        private void Log(string name, object o, params string[] str)
+        {
+            log.Log(name + ": " + o.ToString() + " - " + string.Join(",", str));
+        }
+    }
+}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -2,6 +2,7 @@
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
+using Nethereum.JsonRpc.Client;
 using NUnit.Framework;
 using Utils;
 
@@ -10,6 +11,18 @@ namespace CodexTests.BasicTests
     [TestFixture]
     public class ExampleTests : CodexDistTest
     {
+        [Test]
+        public void A()
+        {
+            var oneMb = GenerateTestFile(1.MB(), "oneMB");
+            var fiveMb = GenerateTestFile(5.MB(), "fiveMb");
+            var tenMb = GenerateTestFile(10.MB(), "tenMb");
+            var hundredMb = GenerateTestFile(100.MB(), "hundredMb");
+            var oneGb = GenerateTestFile(1.GB(), "oneGb");
+
+            var a = 0;
+        }
+
         [Test]
         public void CodexLogExample()
         {
@@ -20,11 +33,41 @@ namespace CodexTests.BasicTests
             var localDatasets = primary.LocalFiles();
             CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
 
+            var nameMap = new Dictionary<string, string>();
+            AddNameMapping(nameMap, primary);
+
+            Get().Replacer = line =>
+            {
+                if (line == null) return null;
+                foreach (var pair in nameMap)
+                {
+                    line = line.Replace(pair.Key, pair.Value);
+                }
+                return line;
+            };
+
+
             var log = Ci.DownloadLog(primary);
 
             log.AssertLogContains("Uploaded file");
         }
 
+
+        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
+        {
+            var name = node.GetName();
+            var info = node.GetDebugInfo();
+            var nodeId = info.Table.LocalNode.NodeId;
+            var peerId = info.Table.LocalNode.PeerId;
+
+            nameMap.Add(nodeId, name);
+            nameMap.Add(peerId, name);
+            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToShortId(peerId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
+        }
+
         [Test]
         public void TwoMetricsExample()
         {

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
 using FileUtils;
@@ -34,10 +35,7 @@ namespace CodexTests.BasicTests
             var numberOfHosts = 5;
             var hosts = StartCodex(numberOfHosts, s => s
                 .WithName("Host")
-                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                {
-                    ContractClock = CodexLogLevel.Trace,
-                })
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Info))
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, m => m
                     .WithInitial(10.Eth(), hostInitialBalance)
@@ -97,6 +95,12 @@ namespace CodexTests.BasicTests
 
             purchaseContract.WaitForStorageContractStarted();
 
+            var availabilities = hosts.Select(h => h.Marketplace.GetAvailabilities()).ToArray();
+            if (availabilities.All(h => h.All(a => a.FreeSpace.SizeInBytes == a.TotalSpace.SizeInBytes)))
+            {
+                Assert.Fail("Host availabilities were not used.");
+            }
+
             var request = GetOnChainStorageRequest(contracts, geth);
             AssertStorageRequest(request, purchase, contracts, client);
             AssertContractSlot(contracts, request, 0);
@@ -108,44 +112,101 @@ namespace CodexTests.BasicTests
         }
 
         [Test]
-        [Ignore("Integrated into MarketplaceExample to speed up testing.")]
-        public void CanDownloadContentFromContractCid()
+        [Combinatorial]
+        public void FindBug(
+            [Values(64)] int numBlocks,
+            [Values(0)] int plusSizeKb,
+            [Values(0)] int plusSizeBytes
+        )
         {
-            var fileSize = 10.MB();
+            var numberOfHosts = 15;
+
+            var hostInitialBalance = 234.Tst();
+            var clientInitialBalance = 100000.Tst();
+            var fileSize = new ByteSize(
+                numBlocks * (64 * 1024) +
+                plusSizeKb * 1024 +
+                plusSizeBytes
+            );
+
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
-            var testFile = CreateFile(fileSize);
+
+            var hosts = StartCodex(numberOfHosts, s => s
+                .WithName("Host")
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Info, CodexLogLevel.Trace))
+                .WithStorageQuota(11.GB())
+                .EnableMarketplace(geth, contracts, m => m
+                    .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsStorageNode()
+                    .AsValidator()));
+
+            foreach (var host in hosts)
+            {
+                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
+
+                var availability = new StorageAvailability(
+                    totalSpace: 10.GB(),
+                    maxDuration: TimeSpan.FromMinutes(30),
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 20.TstWei()
+                );
+                host.Marketplace.MakeStorageAvailable(availability);
+            }
 
             var client = StartCodex(s => s
                 .WithName("Client")
                 .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), 10.Tst())));
+                    .WithInitial(10.Eth(), clientInitialBalance)));
 
-            var uploadCid = client.UploadFile(testFile);
+            var nameMap = new Dictionary<string, string>();
+            AddNameMapping(nameMap, client);
+            foreach (var host in hosts) AddNameMapping(nameMap, host);
 
-            var purchase = new StoragePurchaseRequest(uploadCid)
+            Get().Replacer = line =>
             {
-                PricePerSlotPerSecond = 2.TstWei(),
-                RequiredCollateral = 10.TstWei(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
+                if (line == null) return null;
+                foreach (var pair in nameMap)
+                {
+                    line = line.Replace(pair.Key, pair.Value);
+                }
+                return line;
             };
 
-            var purchaseContract = client.Marketplace.RequestStorage(purchase);
-            var contractCid = purchaseContract.ContentId;
-            Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
+            while (true)
+            {
+                var testFile = CreateFile(fileSize);
+                var uploadCid = client.UploadFile(testFile);
 
-            // Download both from client.
-            testFile.AssertIsEqual(client.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(client.DownloadContent(contractCid));
+                var purchase = new StoragePurchaseRequest(uploadCid)
+                {
+                    PricePerSlotPerSecond = 2.TstWei(),
+                    RequiredCollateral = 10.TstWei(),
+                    MinRequiredNumberOfNodes = 5,
+                    NodeFailureTolerance = 2,
+                    ProofProbability = 5,
+                    Duration = TimeSpan.FromMinutes(20),
+                    Expiry = TimeSpan.FromMinutes(10)
+                };
 
-            // Download both from another node.
-            var downloader = StartCodex(s => s.WithName("Downloader"));
-            testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
+                var purchaseContract = client.Marketplace.RequestStorage(purchase);
+                purchaseContract.WaitForStorageContractStarted();
+            }
+        }
+
+        private void AddNameMapping(Dictionary<string, string> nameMap, ICodexNode node)
+        {
+            var name = node.GetName();
+            var info = node.GetDebugInfo();
+            var nodeId = info.Table.LocalNode.NodeId;
+            var peerId = info.Table.LocalNode.PeerId;
+
+            nameMap.Add(nodeId, name);
+            nameMap.Add(peerId, name);
+            nameMap.Add(CodexUtils.ToShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToShortId(peerId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(nodeId), name);
+            nameMap.Add(CodexUtils.ToNodeIdShortId(peerId), name);
         }
 
         private TrackedFile CreateFile(ByteSize fileSize)

Tests/CodexTests/BasicTests/PyramidTests.cs

@@ -17,7 +17,7 @@ namespace CodexTests.BasicTests
         public void PyramidTest()
         {
             var size = 5.MB();
-            var numberOfLayers = 4;
+            var numberOfLayers = 3;
 
             var bottomLayer = StartLayers(numberOfLayers);
 

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -23,21 +23,23 @@ namespace CodexTests.BasicTests
         }
 
         [Test]
-        [CreateTranscript(nameof(SwarmTest))]
-        public void SwarmTest()
+        public void FindBug()
         {
-            var uploader = StartCodex(s => s.WithName("uploader"));
-            var downloaders = StartCodex(5, s => s.WithName("downloader"));
+            var uploader = StartCodex();
+            var downloaders = StartCodex(10);
 
-            var file = GenerateTestFile(100.MB());
-            var cid = uploader.UploadFile(file);
-
-            var result = Parallel.ForEach(downloaders, d =>
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(15))
             {
-                d.DownloadContent(cid);
-            });
+                var cid = uploader.UploadFile(GenerateTestFile(5.MB()));
 
-            Assert.That(result.IsCompleted);
+                var loop = Parallel.ForEach(downloaders, d =>
+                {
+                    d.DownloadContent(cid);
+                });
+
+                Assert.That(loop.IsCompleted);
+            }
         }
 
         [Test]

Tests/CodexTests/CodexTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/DistTestCore/Configuration.cs

@@ -29,12 +29,12 @@ namespace DistTestCore
         /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace, Func<string?, string?> replacer)
         {
-            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
+            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace, replacer);
         }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace, Func<string?, string?> replacer)
         {
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
@@ -45,6 +45,7 @@ namespace DistTestCore
 
             config.AllowNamespaceOverride = false;
             config.Hooks = hooks;
+            config.Replacer = replacer;
 
             return config;
         }

Tests/DistTestCore/DistTest.cs

@@ -35,7 +35,7 @@ namespace DistTestCore
             fixtureLog = new FixtureLog(logConfig, startTime, deployId);
             statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
+            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix, s => s), configuration.GetFileManagerFolder());
 
             Initialize(fixtureLog);
         }

Tests/DistTestCore/DistTestCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Tests/DistTestCore/TestLifecycle.cs

@@ -22,9 +22,10 @@ namespace DistTestCore
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
+            TestNamespace = testNamespace;
             TestStart = DateTime.UtcNow;
 
-            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
+            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace, InternalReplacer), configuration.GetFileManagerFolder(), timeSet);
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
             this.deployId = deployId;
@@ -32,12 +33,19 @@ namespace DistTestCore
             log.WriteLogTag();
         }
 
+        private string? InternalReplacer(string? arg)
+        {
+            return Replacer(arg);
+        }
+
         public DateTime TestStart { get; }
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public string TestNamespace { get; }
         public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
+        public Func<string?, string?> Replacer { get; set; } = s => s;
 
         public void DeleteAllResources()
         {

Tests/FrameworkTests/FrameworkTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tests/FrameworkTests/OverwatchTranscript/TranscriptTests.cs

@@ -21,6 +21,7 @@ namespace FrameworkTests.OverwatchTranscript
         [Test]
         public void WriteAndRun()
         {
+            // unstable.
             WriteTranscript();
             ReadTranscript();
 

Tools/AutoClient/App.cs

@@ -0,0 +1,40 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class App
+    {
+        public App(Configuration config)
+        {
+            Config = config;
+
+            Log = new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "autoclient")),
+                new ConsoleLog()
+            );
+
+            Generator = CreateGenerator();
+            CidRepo = new CidRepo(config);
+            Performance = new Performance(new LogSplitter(
+                new FileLog(Path.Combine(config.LogPath, "performance")),
+                new ConsoleLog()
+            ));
+        }
+
+        public Configuration Config { get; }
+        public ILog Log { get; }
+        public IFileGenerator Generator { get; }
+        public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
+        public CidRepo CidRepo { get; }
+        public Performance Performance { get; }
+
+        private IFileGenerator CreateGenerator()
+        {
+            if (Config.FileSizeMb > 0)
+            {
+                return new RandomFileGenerator(Config, Log);
+            }
+            return new ImageGenerator(Log);
+        }
+    }
+}

Tools/AutoClient/AutoClient.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/AutoClient/CidRepo.cs

@@ -0,0 +1,85 @@
+namespace AutoClient
+{
+    public class CidRepo
+    {
+        private readonly Random random = new Random();
+        private readonly object _lock = new object();
+        private readonly List<CidEntry> entries = new List<CidEntry>();
+        private readonly Configuration config;
+
+        public CidRepo(Configuration config)
+        {
+            this.config = config;
+        }
+
+        public void Add(string nodeId, string cid, long knownSize)
+        {
+            lock (_lock)
+            {
+                entries.Add(new CidEntry(nodeId, cid, knownSize));
+            }
+        }
+
+        public void AddEncoded(string originalCid, string encodedCid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == originalCid);
+                if (entry == null) return;
+
+                entry.Encoded = encodedCid;
+            }
+        }
+
+        public string? GetForeignCid(string myNodeId)
+        {
+            lock (_lock)
+            {
+                while (true)
+                {
+                    if (!entries.Any()) return null;
+                    var available = entries.Where(e => e.NodeId != myNodeId).ToArray();
+                    if (!available.Any()) return null;
+
+                    var i = random.Next(0, available.Length);
+                    var entry = available[i];
+
+                    if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes)))
+                    {
+                        entries.Remove(entry);
+                    }
+                    else
+                    {
+                        return entry.Cid;
+                    }
+                }
+            }
+        }
+
+        public long? GetSizeForCid(string cid)
+        {
+            lock (_lock)
+            {
+                var entry = entries.SingleOrDefault(e => e.Cid == cid);
+                if (entry == null) return null;
+                return entry.KnownSize;
+            }
+        }
+    }
+
+    public class CidEntry
+    {
+        public CidEntry(string nodeId, string cid, long knownSize)
+        {
+            NodeId = nodeId;
+            Cid = cid;
+            KnownSize = knownSize;
+        }
+
+        public string NodeId { get; }
+        public string Cid { get; }
+        public string Encoded { get; set; } = string.Empty;
+        public long KnownSize { get; }
+        public DateTime CreatedUtc { get; } = DateTime.UtcNow;
+    }
+}

Tools/AutoClient/CodexUser.cs

@@ -0,0 +1,57 @@
+using CodexOpenApi;
+using Logging;
+using Utils;
+
+namespace AutoClient
+{
+    public class CodexUser
+    {
+        private readonly App app;
+        private readonly CodexApi codex;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly List<Purchaser> purchasers = new List<Purchaser>();
+        private Task starterTask = Task.CompletedTask;
+        private readonly string nodeId = Guid.NewGuid().ToString();
+
+        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            this.app = app;
+            this.codex = codex;
+            this.client = client;
+            this.address = address;
+        }
+
+        public void Start(int index)
+        {
+            for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
+            {
+                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+            }
+
+            var delayPerPurchaser =
+                TimeSpan.FromSeconds(10 * index) +
+                TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases;
+
+            starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser));
+        }
+
+        private async Task StartPurchasers(TimeSpan delayPerPurchaser)
+        {
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Start();
+                await Task.Delay(delayPerPurchaser);
+            }
+        }
+
+        public void Stop()
+        {
+            starterTask.Wait();
+            foreach (var purchaser in purchasers)
+            {
+                purchaser.Stop();
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -4,11 +4,8 @@ namespace AutoClient
 {
     public class Configuration
     {
-        [Uniform("codex-host", "ch", "CODEXHOST", false, "Codex Host address. (default 'http://localhost')")]
-        public string CodexHost { get; set; } = "http://localhost";
-
-        [Uniform("codex-port", "cp", "CODEXPORT", false, "port number of Codex API. (8080 by default)")]
-        public int CodexPort { get; set; } = 8080;
+        [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
+        public string CodexEndpoints { get; set; } = "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -22,11 +19,11 @@ namespace AutoClient
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
         public int ContractExpiryMinutes { get; set; } = 15;
 
-        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 5)")]
-        public int NumHosts { get; set; } = 5;
+        [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
+        public int NumHosts { get; set; } = 10;
 
-        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 2)")]
-        public int HostTolerance { get; set; } = 2;
+        [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
+        public int HostTolerance { get; set; } = 5;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
         public int Price { get; set; } = 10;

Tools/AutoClient/ImageGenerator.cs

@@ -11,16 +11,16 @@ namespace AutoClient
 
     public class ImageGenerator : IFileGenerator
     {
-        private LogSplitter log;
+        private readonly ILog log;
 
-        public ImageGenerator(LogSplitter log)
+        public ImageGenerator(ILog log)
         {
             this.log = log;
         }
 
         public async Task<string> Generate()
         {
-            log.Log("Fetching random image from picsum.photos...");
+            log.Debug("Fetching random image from picsum.photos...");
             var httpClient = new HttpClient();
             var thing = await httpClient.GetStreamAsync("https://picsum.photos/3840/2160");
 

Tools/AutoClient/Performance.cs

@@ -0,0 +1,65 @@
+using Logging;
+
+namespace AutoClient
+{
+    public class Performance
+    {
+        private readonly ILog log;
+
+        public Performance(ILog log)
+        {
+            this.log = log;
+        }
+
+        public void DownloadFailed(Exception ex)
+        {
+            Log($"Download failed: {ex}");
+        }
+
+        public void DownloadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Download successful: {bytesPerSecond} bytes per second");
+        }
+
+        public void StorageContractCancelled()
+        {
+            Log("Contract cancelled");
+        }
+
+        public void StorageContractErrored(string error)
+        {
+            Log($"Contract errored: {error}");
+        }
+
+        public void StorageContractFinished()
+        {
+            Log("Contract finished");
+        }
+
+        public void StorageContractStarted()
+        {
+            Log("Contract started");
+        }
+
+        public void UploadFailed(Exception ex)
+        {
+            Log($"Upload failed: {ex}");
+        }
+
+        public void UploadSuccessful(long size, TimeSpan time)
+        {
+            long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
+            if (milliseconds < 1) milliseconds = 1;
+            long bytesPerSecond = 1000 * (size / milliseconds);
+            Log($"Upload successful: {bytesPerSecond} bytes per second");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

Tools/AutoClient/Program.cs

@@ -1,16 +1,20 @@
 using ArgsUniform;
 using AutoClient;
 using CodexOpenApi;
-using Core;
-using Logging;
 using Utils;
 
-public static class Program
+public class Program
 {
+    private readonly App app;
+
+    public Program(Configuration config)
+    {
+        app = new App(config);
+    }
+
     public static async Task Main(string[] args)
     {
         var cts = new CancellationTokenSource();
-        var cancellationToken = cts.Token;
         Console.CancelKeyPress += (sender, args) => cts.Cancel();
 
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
@@ -21,58 +25,70 @@ public static class Program
             throw new Exception("Number of concurrent purchases must be > 0");
         }
 
-        var log = new LogSplitter(
-            new FileLog(Path.Combine(config.LogPath, "autoclient")),
-            new ConsoleLog()
-        );
+        var p = new Program(config);
+        await p.Run();
+    }
+
+    public async Task Run()
+    {
+        var codexUsers = await CreateUsers();
+
+        var i = 0;
+        foreach (var user in codexUsers)
+        {
+            user.Start(i);
+            i++;
+        }
+
+        app.Cts.Token.WaitHandle.WaitOne();
+
+        foreach (var user in codexUsers) user.Stop();
+
+        app.Log.Log("Done");
+    }
+
+    private async Task<CodexUser[]> CreateUsers()
+    {
+        var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
+        var result = new List<CodexUser>();
+
+        foreach (var e in endpointStrs)
+        {
+            result.Add(await CreateUser(e));
+        }
+
+        return result.ToArray();
+    }
+
+    private async Task<CodexUser> CreateUser(string endpoint)
+    {
+        var splitIndex = endpoint.LastIndexOf(':');
+        var host = endpoint.Substring(0, splitIndex);
+        var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
 
         var address = new Address(
-            host: config.CodexHost,
-            port: config.CodexPort
+            host: host,
+            port: port
         );
 
-        log.Log($"Start. Address: {address}");
-
-        var generator = CreateGenerator(config, log);
-
         var client = new HttpClient();
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
-        await CheckCodex(codex, log);
+        app.Log.Log($"Checking Codex at {address}...");
+        await CheckCodex(codex);
+        app.Log.Log("OK");
 
-        var purchasers = new List<Purchaser>();
-        for (var i = 0; i < config.NumConcurrentPurchases; i++)
-        {
-            purchasers.Add(
-                new Purchaser(new LogPrefixer(log, $"({i}) "), client, address, codex, config, generator, cancellationToken)
-            );
-        }
-
-        var delayPerPurchaser = TimeSpan.FromMinutes(config.ContractDurationMinutes) / config.NumConcurrentPurchases;
-        foreach (var purchaser in purchasers) 
-        {
-            purchaser.Start();
-            await Task.Delay(delayPerPurchaser);
-        }
-
-        cancellationToken.WaitHandle.WaitOne();
-
-        log.Log("Done.");
+        return new CodexUser(
+            app,
+            codex,
+            client,
+            address
+        );
     }
 
-    private static IFileGenerator CreateGenerator(Configuration config, LogSplitter log)
+    private async Task CheckCodex(CodexApi codex)
     {
-        if (config.FileSizeMb > 0)
-        {
-            return new RandomFileGenerator(config, log);
-        }
-        return new ImageGenerator(log);
-    }
-
-    private static async Task CheckCodex(CodexApi codex, ILog log)
-    {
-        log.Log("Checking Codex...");
         try
         {
             var info = await codex.GetDebugInfoAsync();
@@ -80,7 +96,7 @@ public static class Program
         }
         catch (Exception ex)
         {
-            log.Log($"Codex not OK: {ex}");
+            app.Log.Error($"Codex not OK: {ex}");
             throw;
         }
     }

Tools/AutoClient/Purchaser.cs

@@ -8,36 +8,76 @@ namespace AutoClient
 {
     public class Purchaser
     {
+        private readonly App app;
+        private readonly string nodeId;
         private readonly ILog log;
         private readonly HttpClient client;
         private readonly Address address;
         private readonly CodexApi codex;
-        private readonly Configuration config;
-        private readonly IFileGenerator generator;
-        private readonly CancellationToken ct;
+        private Task workerTask = Task.CompletedTask;
 
-        public Purchaser(ILog log, HttpClient client, Address address, CodexApi codex, Configuration config, IFileGenerator generator, CancellationToken ct)
+        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
         {
+            this.app = app;
+            this.nodeId = nodeId;
             this.log = log;
             this.client = client;
             this.address = address;
             this.codex = codex;
-            this.config = config;
-            this.generator = generator;
-            this.ct = ct;
         }
 
         public void Start()
         {
-            Task.Run(Worker);
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
         }
 
         private async Task Worker()
         {
-            while (!ct.IsCancellationRequested)
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
             {
-                var pid = await StartNewPurchase();
-                await WaitTillFinished(pid);
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(nodeId);
+            if (cid == null) return;
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+                {
+                    using var fileStream = File.OpenWrite(filename);
+                    var fileResponse = await codex.DownloadNetworkAsync(cid);
+                    fileResponse.Stream.CopyTo(fileStream);
+                }
+                var time = sw.Elapsed;
+                File.Delete(filename);
+                app.Performance.DownloadSuccessful(size.Value, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
             }
         }
 
@@ -50,72 +90,96 @@ namespace AutoClient
 
         private async Task<string> CreateFile()
         {
-            return await generator.Generate();
+            return await app.Generator.Generate();
         }
 
         private async Task<ContentId> UploadFile(string filename)
         {
-            // Copied from CodexNode :/
             using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(nodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
 
-            log.Log($"Uploading file {filename}...");
-            var response = await codex.UploadAsync(fileStream, ct);
+        private async Task<ContentId> UploadStream(FileStream fileStream)
+        {
+            log.Debug($"Uploading file...");
+            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
 
-            log.Log($"Uploaded file. Received contentId: '{response}'.");
+            log.Debug($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
         private async Task<string> RequestStorage(ContentId cid)
         {
-            log.Log("Requesting storage for " + cid.Id);
+            log.Debug("Requesting storage for " + cid.Id);
             var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
             {
-                Collateral = config.RequiredCollateral.ToString(),
-                Duration = (config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = config.NumHosts,
-                Reward = config.Price.ToString(),
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
                 ProofProbability = "15",
-                Tolerance = config.HostTolerance
-            }, ct);
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
 
-            log.Log("Purchase ID: " + result);
+            log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
 
             return result;
         }
 
-        private async Task<string?> GetPurchaseState(string pid)
+        private async Task<string> GetEncodedCid(string pid)
         {
             try
             {
-                // openapi still don't match code.
-                var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-                if (string.IsNullOrEmpty(str)) return null;
-                var sp = JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                log.Log($"Purchase {pid} is {sp.State}");
-                if (!string.IsNullOrEmpty(sp.Error)) log.Log($"Purchase {pid} error is {sp.Error}");
-                return sp.State;
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
             }
-            catch
+            catch (Exception ex)
             {
-                return null;
+                log.Error(ex.ToString());
+                throw;
             }
         }
 
+        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
         private async Task WaitTillFinished(string pid)
         {
-            log.Log("Waiting...");
             try
             {
                 var emptyResponseTolerance = 10;
-                while (true)
+                while (!app.Cts.Token.IsCancellationRequested)
                 {
-                    var status = (await GetPurchaseState(pid))?.ToLowerInvariant();
-                    if (string.IsNullOrEmpty(status))
+                    var purchase = await GetStoragePurchase(pid);
+                    if (purchase == null)
                     {
+                        await FixedShortDelay();
                         emptyResponseTolerance--;
                         if (emptyResponseTolerance == 0)
                         {
@@ -123,19 +187,28 @@ namespace AutoClient
                             await ExpiryTimeDelay();
                             return;
                         }
+                        continue;
                     }
-                    else
+                    var status = purchase.State.ToLowerInvariant();
+                    if (status.Contains("cancel"))
                     {
-                        if (status.Contains("cancel") ||
-                            status.Contains("error") ||
-                            status.Contains("finished"))
-                        {
-                            return;
-                        }
-                        if (status.Contains("started"))
-                        {
-                            await FixedDurationDelay();
-                        }
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (status.Contains("error"))
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (status.Contains("finished"))
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (status.Contains("started"))
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
                     }
 
                     await FixedShortDelay();
@@ -150,17 +223,17 @@ namespace AutoClient
 
         private async Task FixedDurationDelay()
         {
-            await Task.Delay(config.ContractDurationMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task ExpiryTimeDelay()
         {
-            await Task.Delay(config.ContractExpiryMinutes * 60 * 1000, ct);
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
         }
 
         private async Task FixedShortDelay()
         {
-            await Task.Delay(15 * 1000, ct);
+            await Task.Delay(15 * 1000, app.Cts.Token);
         }
     }
 }

Tools/BiblioTech/AdminChecker.cs

@@ -27,9 +27,9 @@ namespace BiblioTech
             return channel.Id == Program.Config.AdminChannelId;
         }
 
-        public ISocketMessageChannel GetAdminChannel()
+        public async Task SendInAdminChannel(string msg)
         {
-            return adminChannel;
+            await adminChannel.SendMessageAsync(msg);
         }
 
         public void SetAdminChannel(ISocketMessageChannel adminChannel)

Tools/BiblioTech/BaseCommand.cs

@@ -1,6 +1,7 @@
 using Discord.WebSocket;
 using BiblioTech.Options;
 using Discord;
+using k8s.KubeConfigModels;
 
 namespace BiblioTech
 {
@@ -25,16 +26,13 @@ namespace BiblioTech
             catch (Exception ex)
             {
                 var msg = "Failed with exception: " + ex;
-                if (IsInAdminChannel(command))
-                {
-                    await command.FollowupAsync(msg.Substring(0, Math.Min(1900, msg.Length)));
-                }
-                else
-                {
-                    await command.FollowupAsync("Something failed while trying to do that...", ephemeral: true);
-                    await Program.AdminChecker.GetAdminChannel().SendMessageAsync(msg);
-                }
                 Program.Log.Error(msg);
+
+                if (!IsInAdminChannel(command))
+                {
+                    await command.FollowupAsync("Something failed while trying to do that... (error details posted in admin channel)", ephemeral: true);
+                }
+                await Program.AdminChecker.SendInAdminChannel(msg);
             }
         }
 
@@ -62,5 +60,20 @@ namespace BiblioTech
             if (IsSenderAdmin(context.Command) && targetUser != null) return targetUser;
             return context.Command.User;
         }
+
+        protected string Mention(SocketUser user)
+        {
+            return Mention(user.Id);
+        }
+
+        protected string Mention(IUser user)
+        {
+            return Mention(user.Id);
+        }
+
+        protected string Mention(ulong userId)
+        {
+            return $"<@{userId}>";
+        }
     }
 }

Tools/BiblioTech/BiblioTech.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/BiblioTech/Commands/GetBalanceCommand.cs

@@ -28,6 +28,7 @@ namespace BiblioTech.Commands
             if (addr == null)
             {
                 await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' but address has not been set.");
                 return;
             }
 

Tools/BiblioTech/Commands/MintCommand.cs

@@ -28,6 +28,7 @@ namespace BiblioTech.Commands
             if (addr == null)
             {
                 await context.Followup($"No address has been set for this user. Please use '/{userAssociateCommand.Name}' to set it first.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' but address has not been set.");
                 return;
             }
 
@@ -42,9 +43,17 @@ namespace BiblioTech.Commands
                 mintedTokens = ProcessTokens(contracts, addr, report);
             });
 
+            var reportLine = string.Join(Environment.NewLine, report);
             Program.UserRepo.AddMintEventForUser(userId, addr, sentEth, mintedTokens);
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(userId)} used '/{Name}' successfully. ({reportLine})");
 
-            await context.Followup(string.Join(Environment.NewLine, report));
+            await context.Followup(reportLine);
+        }
+
+        private string Format<T>(Transaction<T>? transaction)
+        {
+            if (transaction == null) return "-";
+            return transaction.ToString();
         }
 
         private Transaction<TestToken>? ProcessTokens(ICodexContracts contracts, EthAddress addr, List<string> report)

Tools/BiblioTech/Commands/UserAssociateCommand.cs

@@ -1,4 +1,8 @@
 using BiblioTech.Options;
+using Discord;
+using GethPlugin;
+using k8s.KubeConfigModels;
+using NBitcoin.Secp256k1;
 
 namespace BiblioTech.Commands
 {
@@ -23,30 +27,56 @@ namespace BiblioTech.Commands
         protected override async Task Invoke(CommandContext context)
         {
             var user = GetUserFromCommand(optionalUser, context);
-            var data = await ethOption.Parse(context);
-            if (data == null) return;
+            var newAddress = await ethOption.Parse(context);
+            if (newAddress == null) return;
 
             var currentAddress = Program.UserRepo.GetCurrentAddressForUser(user);
             if (currentAddress != null && !IsSenderAdmin(context.Command))
             {
                 await context.Followup($"You've already set your Ethereum address to {currentAddress}.");
+                await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but already has an address set. ({currentAddress})");
                 return;
             }
 
-            var result = Program.UserRepo.AssociateUserWithAddress(user, data);
-            if (result)
+            var result = Program.UserRepo.AssociateUserWithAddress(user, newAddress);
+            switch (result)
             {
-                await context.Followup(new string[]
-                {
+                case SetAddressResponse.OK:
+                    await ResponseOK(context, user, newAddress);
+                    break;
+                case SetAddressResponse.AddressAlreadyInUse:
+                    await ResponseAlreadyUsed(context, user, newAddress);
+                    break;
+                case SetAddressResponse.CreateUserFailed:
+                    await ResponseCreateUserFailed(context, user);
+                    break;
+                default:
+                    throw new Exception("Unknown SetAddressResponse mode");
+            }
+        }
+
+        private async Task ResponseCreateUserFailed(CommandContext context, IUser user)
+        {
+            await context.Followup("Internal error. Error details sent to admin.");
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but failed to create new user.");
+        }
+
+        private async Task ResponseAlreadyUsed(CommandContext context, IUser user, EthAddress newAddress)
+        {
+            await context.Followup("This address is already in use by another user.");
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' but the provided address is already in use by another user. (address: {newAddress})");
+        }
+
+        private async Task ResponseOK(CommandContext context, IUser user, GethPlugin.EthAddress newAddress)
+        {
+            await context.Followup(new string[]
+{
                     "Done! Thank you for joining the test net!",
-                    "By default, the bot will @-mention you with test-net reward related notifications.",
+                    "By default, the bot will @-mention you with test-net related notifications.",
                     $"You can enable/disable this behavior with the '/{notifyCommand.Name}' command."
-                });
-            }
-            else
-            {
-                await context.Followup("That didn't work.");
-            }
+});
+
+            await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' successfully. ({newAddress})");
         }
     }
 }

Tools/BiblioTech/Rewards/ChainEventsSender.cs

@@ -31,7 +31,7 @@ namespace BiblioTech.Rewards
                         {
                             var @event = ApplyReplacements(users, e);
                             await eventsChannel.SendMessageAsync(@event);
-                            await Task.Delay(3000);
+                            await Task.Delay(1000);
                         }
                     }
                 });

Tools/BiblioTech/Transaction.cs

@@ -10,5 +10,11 @@
 
         public T TokenAmount { get; }
         public string TransactionHash { get; }
+
+        public override string ToString()
+        {
+            if (TokenAmount == null) return "NULL";
+            return TokenAmount.ToString()!;
+        }
     }
 }

Tools/BiblioTech/UserRepo.cs

@@ -10,7 +10,7 @@ namespace BiblioTech
         private readonly object repoLock = new object();
         private readonly Dictionary<ulong, UserData> cache = new Dictionary<ulong, UserData>();
 
-        public bool AssociateUserWithAddress(IUser user, EthAddress address)
+        public SetAddressResponse AssociateUserWithAddress(IUser user, EthAddress address)
         {
             lock (repoLock)
             {
@@ -134,18 +134,19 @@ namespace BiblioTech
             return null;
         }
 
-        private bool SetUserAddress(IUser user, EthAddress? address)
+        private SetAddressResponse SetUserAddress(IUser user, EthAddress? address)
         {
             if (GetUserDataForAddress(address) != null)
             {
-                return false;
+                return SetAddressResponse.AddressAlreadyInUse;
             }
 
             var userData = GetOrCreate(user);
+            if (userData == null) return SetAddressResponse.CreateUserFailed;
             userData.CurrentAddress = address;
             userData.AssociateEvents.Add(new UserAssociateAddressEvent(DateTime.UtcNow, address));
             SaveUserData(userData);
-            return true;
+            return SetAddressResponse.OK;
         }
 
         private void SetUserNotification(IUser user, bool notifyEnabled)
@@ -245,4 +246,11 @@ namespace BiblioTech
             }
         }
     }
+
+    public enum SetAddressResponse
+    {
+        OK,
+        AddressAlreadyInUse,
+        CreateUserFailed
+    }
 }

Tools/CodexNetDeployer/CodexNetDeployer.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/KeyMaker/KeyMaker.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>

Tools/MarketInsights/AverageHistory.cs

@@ -23,7 +23,7 @@ namespace MarketInsights
 
         public MarketTimeSegment[] Segments { get; private set; } = Array.Empty<MarketTimeSegment>();
 
-        public Task OnNewSegment(TimeRange timeRange)
+        public Task<TimeSegmentResponse> OnNewSegment(TimeRange timeRange)
         {
             var contribution = BuildContribution(timeRange);
             contributions.Add(contribution);
@@ -35,7 +35,7 @@ namespace MarketInsights
 
             Segments = contributions.ToArray();
 
-            return Task.CompletedTask;
+            return Task.FromResult(TimeSegmentResponse.OK);
         }
 
         private MarketTimeSegment BuildContribution(TimeRange timeRange)

Tools/MarketInsights/Dockerfile

@@ -3,8 +3,6 @@
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
 USER app
 WORKDIR /app
-EXPOSE 8080
-EXPOSE 8081
 
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release

Tools/MarketInsights/MarketInsights.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>ae71e621-bb16-41b2-b6f3-c597d2d21157</UserSecretsId>

Tools/MarketInsights/Program.cs

@@ -1,6 +1,4 @@
 using ArgsUniform;
-using Microsoft.Extensions.Options;
-using Nethereum.Model;
 using System.Reflection;
 
 namespace MarketInsights
@@ -28,6 +26,14 @@ namespace MarketInsights
 
             var builder = WebApplication.CreateBuilder(args);
 
+            var listenPort = Environment.GetEnvironmentVariable("APIPORT");
+            if (string.IsNullOrEmpty(listenPort)) listenPort = "31090";
+
+            builder.WebHost.ConfigureKestrel((context, options) =>
+            {
+                options.ListenAnyIP(Convert.ToInt32(listenPort));
+            });
+
             builder.Services.AddSingleton(appState);
 
             builder.Services.AddControllers();
@@ -54,6 +60,8 @@ namespace MarketInsights
 
             app.MapControllers();
 
+            Console.WriteLine("MarketInsights listening on port " + listenPort);
+
             updater.Run();
             app.Run();
         }

Tools/TestClusterStarter/TestClusterStarter.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/TestNetRewarder/EventsFormatter.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin;
 using CodexContractsPlugin.ChainMonitor;
 using GethPlugin;
+using System.Globalization;
 using System.Numerics;
 using Utils;
 
@@ -76,7 +77,7 @@ namespace TestNetRewarder
 
         private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
         {
-            var blockNumber = $"[{requestEvent.Block.BlockNumber}]";
+            var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
             var title = $"{blockNumber} **{eventName}** `{requestEvent.Request.Request.Id}`";
             AddBlock(title, content);
         }
@@ -107,6 +108,11 @@ namespace TestNetRewarder
             ) + nl + nl;
         }
 
+        private string FormatDateTime(DateTime utc)
+        {
+            return utc.ToString("yyyy-MM-dd HH:mm:ss UTC", CultureInfo.InvariantCulture);
+        }
+
         private string BigIntToDuration(BigInteger big)
         {
             var span = TimeSpan.FromSeconds((int)big);

Tools/TestNetRewarder/Processor.cs

@@ -31,19 +31,18 @@ namespace TestNetRewarder
             chainState = new ChainState(log, contracts, handler, config.HistoryStartUtc);
         }
 
-        public async Task OnNewSegment(TimeRange timeRange)
+        public async Task<TimeSegmentResponse> OnNewSegment(TimeRange timeRange)
         {
             try
             {
-                chainState.Update(timeRange.To);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var numberOfChainEvents = await ProcessEvents(timeRange);
+                var duration = sw.Elapsed;
 
-                var events = eventsFormatter.GetEvents();
-
-                var request = builder.Build(events);
-                if (request.HasAny())
-                {
-                    await client.SendRewards(request);
-                }
+                if (numberOfChainEvents == 0) return TimeSegmentResponse.Underload;
+                if (numberOfChainEvents > 10) return TimeSegmentResponse.Overload;
+                if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Overload;
+                return TimeSegmentResponse.OK;
             }
             catch (Exception ex)
             {
@@ -53,5 +52,19 @@ namespace TestNetRewarder
                 throw;
             }
         }
+
+        private async Task<int> ProcessEvents(TimeRange timeRange)
+        {
+            var numberOfChainEvents = chainState.Update(timeRange.To);
+
+            var events = eventsFormatter.GetEvents();
+
+            var request = builder.Build(events);
+            if (request.HasAny())
+            {
+                await client.SendRewards(request);
+            }
+            return numberOfChainEvents;
+        }
     }
 }

Tools/TestNetRewarder/TestNetRewarder.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Tools/TestNetRewarder/TimeSegmenter.cs

@@ -5,15 +5,24 @@ namespace TestNetRewarder
 {
     public interface ITimeSegmentHandler
     {
-        Task OnNewSegment(TimeRange timeRange);
+        Task<TimeSegmentResponse> OnNewSegment(TimeRange timeRange);
+    }
+
+    public enum TimeSegmentResponse
+    {
+        OK,
+        Underload,
+        Overload
     }
 
     public class TimeSegmenter
     {
+        private const int maxSegmentMult = 50;
         private readonly ILog log;
         private readonly ITimeSegmentHandler handler;
         private readonly TimeSpan segmentSize;
         private DateTime latest;
+        private int currentSegmentMult = 1;
 
         public TimeSegmenter(ILog log, TimeSpan segmentSize, DateTime historyStartUtc, ITimeSegmentHandler handler)
         {
@@ -30,19 +39,47 @@ namespace TestNetRewarder
 
         public async Task ProcessNextSegment()
         {
-            var end = latest + segmentSize;
+            var end = GetNewSegmentEnd();
             IsRealtime = await WaitUntilTimeSegmentInPast(end);
 
             if (Program.CancellationToken.IsCancellationRequested) return;
 
             var postfix = "(Catching up...)";
             if (IsRealtime) postfix = "(Real-time)";
-            log.Log($"Time segment [{latest} to {end}] {postfix}");
+            log.Log($"Time segment [{latest} to {end}] {postfix}({currentSegmentMult}x)");
             
             var range = new TimeRange(latest, end);
             latest = end;
 
-            await handler.OnNewSegment(range);
+            var response = await handler.OnNewSegment(range);
+            HandleResponse(response);
+        }
+
+        private DateTime GetNewSegmentEnd()
+        {
+            if (IsRealtime) return latest + segmentSize;
+            var segment = segmentSize * currentSegmentMult;
+            var end = latest + segment;
+            if (end > DateTime.UtcNow) return DateTime.UtcNow + segmentSize;
+            return end;
+        }
+
+        private void HandleResponse(TimeSegmentResponse response)
+        {
+            switch (response)
+            {
+                case TimeSegmentResponse.OK:
+                    if (currentSegmentMult > 1) currentSegmentMult--;
+                    break;
+                case TimeSegmentResponse.Underload:
+                    if (currentSegmentMult < maxSegmentMult) currentSegmentMult++;
+                    break;
+                case TimeSegmentResponse.Overload:
+                    currentSegmentMult = 1;
+                    break;
+                default:
+                    throw new Exception("Unknown response type: " + response);
+            }
         }
 
         private async Task<bool> WaitUntilTimeSegmentInPast(DateTime end)
@@ -52,6 +89,7 @@ namespace TestNetRewarder
             var now = DateTime.UtcNow;
             while (end > now)
             {
+                currentSegmentMult = 1;
                 var delay = (end - now) + TimeSpan.FromSeconds(3);
                 await Task.Delay(delay, Program.CancellationToken);
                 return true;

Tools/TranscriptAnalysis/TranscriptAnalysis.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

cs-codex-dist-testing.sln

@@ -74,7 +74,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OverwatchTranscript", "Fram
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools\TranscriptAnalysis\TranscriptAnalysis.csproj", "{C0EEBD32-23CB-45EC-A863-79FB948508C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MarketInsights", "Tools\MarketInsights\MarketInsights.csproj", "{004614DF-1C65-45E3-882D-59AE44282573}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution