Sets up tests for symmetric, asymmetric and binary speed checks

.github/workflows/run-release-tests.yaml

@@ -109,6 +109,7 @@ jobs:
           # Get logs
           while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
             echo "Show ${pod} logs ..."
+            echo "----"
             kubectl logs $pod -n $namespace -f || true
             sleep 1
           done

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
@@ -19,7 +20,7 @@ namespace GethConnector
                 return null;
             }
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
 
             var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
 

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
             handler.Log(stream);
         }
     }

Framework/KubernetesWorkflow/K8sController.cs

@@ -906,7 +906,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }

Framework/KubernetesWorkflow/LogHandler.cs

@@ -25,9 +25,9 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
             var msg = $"{description} -->> {LogFile.FullFilename}";
             sourceLog.Log(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -127,7 +127,7 @@ namespace KubernetesWorkflow
         {
             var msg = $"Downloading container log for '{container.Name}'";
             log.Log(msg);
-            var logHandler = new WriteToFileLogHandler(log, msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
 
             K8s(controller =>
             {

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -72,9 +72,13 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/LogFile.cs

@@ -1,4 +1,6 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
@@ -49,7 +51,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/Utils/Retry.cs

@@ -98,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,7 +1,7 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,9 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,6 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using BlockchainUtils;
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -110,7 +110,20 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -221,8 +234,15 @@ namespace CodexPlugin
                 .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
+        public static Address? UploaderOverride { get; set; } = null;
+        public static Address? DownloaderOverride { get; set; } = null;
+
         private Address GetAddress()
         {
+            if (GetName().ToLowerInvariant().Contains("upload") && UploaderOverride != null) return UploaderOverride;
+            if (GetName().ToLowerInvariant().Contains("download") && DownloaderOverride != null) return DownloaderOverride;
+
+
             return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -48,6 +48,11 @@ namespace CodexPlugin
         public string Message { get; set; } = string.Empty;
         public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
 
+        public override string ToString()
+        {
+            return Message;
+        }
+
         /// <summary>
         /// After too much time spent cursing at regexes, here's what I got:
         /// Parses input string into 'key=value' pair, considerate of quoted (") values.

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -22,7 +22,14 @@ namespace CodexPlugin
         ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
         LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
@@ -151,7 +158,7 @@ namespace CodexPlugin
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
         {
-            return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
         }
 
         public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
@@ -184,10 +191,20 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
+            return DownloadContentT(contentId, fileLabel).Item1;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, onFailure, fileLabel).Item1;
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, DoNothing, fileLabel);
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
             hooks.OnFileDownloading(contentId);
@@ -200,16 +217,33 @@ namespace CodexPlugin
             transferSpeeds.AddDownloadSample(size, measurement);
             hooks.OnFileDownloaded(size, contentId);
 
-            return file;
+            return (file, measurement);
         }
 
         public LocalDataset DownloadStreamless(ContentId cid)
         {
+            Log($"Downloading streamless '{cid}' (no-wait)");
             return CodexAccess.DownloadStreamless(cid);
         }
 
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = CodexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
+        }
+
         public LocalDataset DownloadManifestOnly(ContentId cid)
         {
+            Log($"Downloading manifest-only '{cid}'");
             return CodexAccess.DownloadManifestOnly(cid);
         }
 
@@ -321,6 +355,39 @@ namespace CodexPlugin
             }
         }
 
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
+        }
+
         private void EnsureMarketplace()
         {
             if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -21,6 +21,14 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
+            };
+        }
+
         public LocalDatasetList Map(CodexOpenApi.DataList dataList)
         {
             return new LocalDatasetList
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -188,6 +205,18 @@ namespace CodexPlugin
             };
         }
 
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
         private JArray JArray(IDictionary<string, object> map, string name)
         {
             return (JArray)map[name];
@@ -243,4 +272,42 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -38,6 +38,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -15,6 +15,7 @@ namespace CodexPlugin
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
         void WaitForStorageContractFinished(ICodexContracts contracts);
+        void WaitForContractFailed();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -85,6 +86,17 @@ namespace CodexPlugin
             Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
         }
 
+        public void WaitForContractFailed()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "failed");
+        }
+
         public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
             return codexAccess.GetPurchaseStatus(purchaseId);

ProjectPlugins/GethPlugin/CoreInterfaceExtensions.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -9,15 +10,15 @@ namespace GethPlugin
             return Plugin(ci).DeployGeth(setup);
         }
 
-        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment)
+        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment, BlockCache blockCache)
         {
-            return Plugin(ci).WrapGethDeployment(deployment);
+            return Plugin(ci).WrapGethDeployment(deployment, blockCache);
         }
 
-        public static IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)
+        public static IGethNode StartGethNode(this CoreInterface ci, BlockCache blockCache, Action<IGethSetup> setup)
         {
             var deploy = DeployGeth(ci, setup);
-            return WrapGethDeployment(ci, deploy);
+            return WrapGethDeployment(ci, deploy, blockCache);
         }
 
         private static GethPlugin Plugin(CoreInterface ci)

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,11 +1,11 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow.Types;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -34,10 +34,12 @@ namespace GethPlugin
     public class DeploymentGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
 
-        public DeploymentGethNode(ILog log, GethDeployment startResult)
+        public DeploymentGethNode(ILog log, BlockCache blockCache, GethDeployment startResult)
         {
             this.log = log;
+            this.blockCache = blockCache;
             StartResult = startResult;
         }
 
@@ -60,7 +62,7 @@ namespace GethPlugin
             var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
-            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }
@@ -68,6 +70,7 @@ namespace GethPlugin
     public class CustomGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string gethHost;
         private readonly int gethPort;
         private readonly string privateKey;
@@ -75,9 +78,10 @@ namespace GethPlugin
         public GethDeployment StartResult => throw new NotImplementedException();
         public RunningContainer Container => throw new NotImplementedException();
 
-        public CustomGethNode(ILog log, string gethHost, int gethPort, string privateKey)
+        public CustomGethNode(ILog log, BlockCache blockCache, string gethHost, int gethPort, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.gethHost = gethHost;
             this.gethPort = gethPort;
             this.privateKey = privateKey;
@@ -90,7 +94,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var creator = new NethereumInteractionCreator(log, gethHost, gethPort, privateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, gethHost, gethPort, privateKey);
             return creator.CreateWorkflow();
         }
     }

ProjectPlugins/GethPlugin/GethPlugin.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -36,10 +37,10 @@ namespace GethPlugin
             return starter.StartGeth(startupConfig);
         }
 
-        public IGethNode WrapGethDeployment(GethDeployment startResult)
+        public IGethNode WrapGethDeployment(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return starter.WrapGethContainer(startResult);
+            return starter.WrapGethContainer(startResult, blockCache);
         }
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow;
 
 namespace GethPlugin
@@ -41,10 +42,10 @@ namespace GethPlugin
             return new GethDeployment(containers, discoveryPort, httpPort, wsPort, account, pubKey);
         }
 
-        public IGethNode WrapGethContainer(GethDeployment startResult)
+        public IGethNode WrapGethContainer(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return new DeploymentGethNode(tools.GetLog(), startResult);
+            return new DeploymentGethNode(tools.GetLog(), blockCache, startResult);
         }
 
         private void Log(string msg)

SpeedCheckTests/Parallelism.cs

@@ -0,0 +1,6 @@
+using NUnit.Framework;
+
+[assembly: LevelOfParallelism(1)]
+namespace CodexTests
+{
+}

SpeedCheckTests/SpeedCheckTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="nunit" Version="3.13.3" />
+		<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+		<ProjectReference Include="..\Tests\DistTestCore\DistTestCore.csproj" />
+		<ProjectReference Include="..\Tests\ExperimentalTests\ExperimentalTests.csproj" />
+	</ItemGroup>
+</Project>

SpeedCheckTests/SpeedTest.cs

@@ -0,0 +1,107 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System.Diagnostics;
+using System.Drawing;
+using Utils;
+
+namespace SpeedCheckTests
+{
+    [TestFixture]
+    public class SpeedTest : CodexDistTest
+    {
+        [Test]
+        public void Symmetric()
+        {
+            // Symmetric: A node always sends a reply to every message it receives.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var uploader = StartCodex(s => s.WithName("SymUploader"));
+            var downloader = StartCodex(s => s.WithName("SymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Symmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)), 
+                $"Symmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Asymmetric()
+        {
+            // Asymmetric: A node does not always send a reply when a message is received.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-s";
+
+            var uploader = StartCodex(s => s.WithName("AsymUploader"));
+            var downloader = StartCodex(s => s.WithName("AsymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Asymmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Asymmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Binary()
+        {
+            // Docker image not used: Here for api check.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var binary = "C:\\Projects\\nim-codex\\build\\codex.exe";
+            if (!File.Exists(binary)) throw new Exception("TODO: Update binary path");
+
+            var uploadInfo = new ProcessStartInfo
+            {
+                FileName = binary,
+                Arguments = "--data-dir=upload_data " +
+                "--api-port=8081 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8091 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8071",
+                UseShellExecute = true,
+            };
+
+            var uploadProcess = Process.Start(uploadInfo);
+
+            Thread.Sleep(5000);
+            if (uploadProcess == null || uploadProcess.HasExited) throw new Exception("Node exited.");
+
+            CodexAccess.UploaderOverride = new Address("http://localhost", 8081);
+            var uploader = StartCodex(s => s.WithName("BinaryUploader"));
+            var spr = uploader.GetSpr();
+            
+            var downloadProcess = Process.Start(binary,
+                "--data-dir=download_data " +
+                "--api-port=8082 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8092 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8072 " +
+                "--bootstrap-node=" + spr
+            );
+
+            CodexAccess.DownloaderOverride = new Address("http://localhost", 8082);
+            var downloader = StartCodex(s => s.WithName("BinaryDownloader"));
+
+            var timeTaken = PerformTest(uploader, downloader);
+
+            uploadProcess.Kill();
+            downloadProcess.Kill();
+
+            Console.WriteLine($"Binary time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Binary: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        private TimeSpan PerformTest(ICodexNode uploader, ICodexNode downloader)
+        {
+            var testFile = GenerateTestFile(100.MB());
+            var contentId = uploader.UploadFile(testFile);
+            var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
+            return timeTaken;
+        }
+    }
+}

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -130,7 +130,7 @@ namespace ContinuousTests
                 var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
                     effectiveEnd, openingLine);
             }
         }

Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs

@@ -1,5 +1,7 @@
-using CodexTests;
+using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System.Drawing;
 using Utils;
 
 namespace CodexReleaseTests.DataTests
@@ -13,31 +15,17 @@ namespace CodexReleaseTests.DataTests
             var uploader = StartCodex();
             var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
 
-            var file = GenerateTestFile(10.MB());
-            var size = file.GetFilesize().SizeInBytes;
+            var size = 10.MB();
+            var file = GenerateTestFile(size);
             var cid = uploader.UploadFile(file);
 
             var startSpace = downloader.Space();
             var start = DateTime.UtcNow;
-            var localDataset = downloader.DownloadStreamless(cid);
+            var localDataset = downloader.DownloadStreamlessWait(cid, size);
 
             Assert.That(localDataset.Cid, Is.EqualTo(cid));
             Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
 
-            // TODO: We have no way to inspect the status or progress of the download.
-            // We use local space information to estimate.
-            var retry = new Retry("Checking local space",
-                maxTimeout: TimeSpan.FromMinutes(2),
-                sleepAfterFail: TimeSpan.FromSeconds(3),
-                onFail: f => { });
-
-            retry.Run(() =>
-            {
-                var space = downloader.Space();
-                var expected = startSpace.FreeBytes - size;
-                if (space.FreeBytes > expected) throw new Exception("Expected free space not reached.");
-            });
-
             // Stop the uploader node and verify that the downloader has the data.
             uploader.Stop(waitTillStopped: true);
             var downloaded = downloader.DownloadContent(cid);

Tests/CodexReleaseTests/DataTests/SwarmTest.cs

@@ -9,14 +9,16 @@ namespace CodexReleaseTests.DataTests
     [TestFixture]
     public class SwarmTests : AutoBootstrapDistTest
     {
-        private const int NumberOfNodes = 2;
-        private const int FileSizeMb = 2;
-
         [Test]
-        public void SmallSwarm()
+        [Combinatorial]
+        public void SmallSwarm(
+            [Values(2)] int numberOfNodes,
+            [Values(10)] int filesizeMb
+        )
         {
-            var nodes = StartCodex(NumberOfNodes);
-            var files = nodes.Select(UploadUniqueFilePerNode).ToArray();
+            var filesize = filesizeMb.MB();
+            var nodes = StartCodex(numberOfNodes);
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
 
             var tasks = ParallelDownloadEachFile(nodes, files);
             Task.WaitAll(tasks);
@@ -24,11 +26,28 @@ namespace CodexReleaseTests.DataTests
             AssertAllFilesDownloadedCorrectly(files);
         }
 
-        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node)
+        [Test]
+        [Combinatorial]
+        public void StreamlessSmallSwarm(
+            [Values(2)] int numberOfNodes,
+            [Values(10)] int filesizeMb
+        )
         {
-            var file = GenerateTestFile(FileSizeMb.MB());
+            var filesize = filesizeMb.MB();
+            var nodes = StartCodex(numberOfNodes);
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
+
+            var tasks = ParallelStreamlessDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesStreamlesslyDownloadedCorrectly(nodes, files);
+        }
+
+        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize)
+        {
+            var file = GenerateTestFile(fileSize);
             var cid = node.UploadFile(file);
-            return new SwarmTestNetworkFile(file, cid);
+            return new SwarmTestNetworkFile(node, fileSize, file, cid);
         }
 
         private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
@@ -43,6 +62,18 @@ namespace CodexReleaseTests.DataTests
             return tasks.ToArray();
         }
 
+        private Task[] ParallelStreamlessDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                tasks.Add(StartStreamlessDownload(node, files));
+            }
+
+            return tasks.ToArray();
+        }
+
         private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
         {
             return Task.Run(() =>
@@ -68,6 +99,31 @@ namespace CodexReleaseTests.DataTests
             });
         }
 
+        private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files)
+        {
+            return Task.Run(() =>
+            {
+                var remaining = files.ToList();
+
+                while (remaining.Count > 0)
+                {
+                    var file = remaining.PickOneRandom();
+                    if (file.Uploader.GetName() != node.GetName())
+                    {
+                        try
+                        {
+                            var startSpace = node.Space();
+                            node.DownloadStreamlessWait(file.Cid, file.OriginalSize);
+                        }
+                        catch (Exception ex)
+                        {
+                            file.Error = ex;
+                        }
+                    }
+                }
+            });
+        }
+
         private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
         {
             foreach (var file in files)
@@ -83,14 +139,34 @@ namespace CodexReleaseTests.DataTests
             }
         }
 
+        private void AssertAllFilesStreamlesslyDownloadedCorrectly(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var totalFilesSpace = 0.Bytes();
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes);
+            }
+            
+            foreach (var node in nodes)
+            {
+                var currentSpace = node.Space();
+                Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes));
+            }
+        }
+
         private class SwarmTestNetworkFile
         {
-            public SwarmTestNetworkFile(TrackedFile original, ContentId cid)
+            public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid)
             {
+                Uploader = uploader;
+                OriginalSize = originalSize;
                 Original = original;
                 Cid = cid;
             }
 
+            public ICodexNode Uploader { get; }
+            public ByteSize OriginalSize { get; }
             public TrackedFile Original { get; }
             public ContentId Cid { get; }
             public object Lock { get; } = new object();

Tests/CodexReleaseTests/DataTests/TwoClientTest.cs

@@ -14,8 +14,67 @@ namespace CodexReleaseTests.DataTests
     public class TwoClientTests : CodexDistTest
     {
         [Test]
-        public void TwoClientTest()
+        [Combinatorial]
+        public void TwoClientTest(
+            [Values(
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-2", // S don't send cancel-presence messages
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-1", // F ignore cancel-presence messages
+            //"codexstorage/nim-codex:sha-4b5c355-dist-tests", // F unmodified
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-3", // F same as 1 but logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-4", // S no cancel-presence-msg, no fromCancel field
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-5", // F all-presence = cancel? return from handler
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-6", // F no cancel-presence-msg, but if any cancel send empty presence msg
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-7", // F same but logs outgoing empty presence message. (msg is empty structure)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-8", // crashes F? eventtimelogging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-9", // crashes S? eventtimelogging + no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-10", // F eventtimelogging (should be fast)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-11", // S eventtimelogging + no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-12", // F upload and download event logging (should be fast)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-13", // S same but with no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:peerselecta-1", // F PR update (yes cancel-presence-msg)
+            //"thatbenbierens/nim-codex:peerselecta-2", // S PR update (no cancel-presence-msg)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-14", // F new logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-15", // S new logging
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-16-f", // F more logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-16-s", // S more logging
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-17-f", // F "tick" every 100 milliseconds
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-17-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-18-f", // F "tick" every 10 milliseconds
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-18-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-19-f", // F sending/sent/received logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-19-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-20-f", // F sending/sent/received logs + number
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-20-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-21-f", // F libp2p lpchannel.write logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-21-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-22-f", // F chronos stream write logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-22-s", // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-23-f", // F chronos stream write logs in libp2p hand-off
+            "thatbenbierens/nim-codex:blkex-cancelpresence-23-s", // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-25-f", // F chronos stream write logs in libp2p hand-off with ticks
+            "thatbenbierens/nim-codex:blkex-cancelpresence-25-s",  // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-27-f", // F chronos stream write logs in libp2p hand-off with ticks adds names
+            "thatbenbierens/nim-codex:blkex-cancelpresence-27-s"  // S same but slow
+            )] string img
+        )
         {
+            CodexContainerRecipe.DockerImageOverride = img;
+
             var uploader = StartCodex(s => s.WithName("Uploader"));
             var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
 
@@ -23,25 +82,111 @@ namespace CodexReleaseTests.DataTests
         }
 
         [Test]
-        [Ignore("Location selection is currently unavailable.")]
-        public void TwoClientsTwoLocationsTest()
+        public void ParseLogs()
         {
-            var locations = Ci.GetKnownLocations();
-            if (locations.NumberOfLocations < 2)
+            var path = "d:\\Dev\\cs-codex-dist-tests\\Tests\\CodexReleaseTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2025-01\\09\\13-58-28Z_TwoClientTests\\";
+            var file1 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-f]_000001_Downloader1.log");
+            var file2 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-f]_000000_Uploader0.log");
+            var file3 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-s]_000001_Downloader1.log");
+            var file4 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-s]_000000_Uploader0.log");
+
+            var lines = File.ReadAllLines(file3);
+            var clines = new List<CodexLogLine>();
+            foreach (var line in lines)
             {
-                Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
-                return;
+                var cline = CodexLogLine.Parse(line);
+                if (cline != null) clines.Add(cline);
             }
 
-            var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
-            var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
+            var gaps = new List<Gap>();
+            for (var i = 0; i < clines.Count; i++)
+            {
+                var line = clines[i];
+
+
+                // todo:
+//TRC 2025-01-09 13:59:14.501+00:00 chronosread                                topics="libp2p chronosstream custom" tid=1 ticks=424485 name=ChronosStream count=32669
+//TRC 2025-01-09 13:59:14.501+00:00 chronosread                                topics="libp2p chronosstream custom" tid=1 ticks=600 name=ChronosStream count=32670
+//TRC 2025-01-09 13:59:14.501+00:00 readOnce                                   topics="libp2p mplexchannel custom" tid=1 s=16U*uBBR7j:677fd62fe0c5bd152c675e42:677fd62ff7548faf70a27174 bytes=1 count=32671
+//TRC 2025-01-09 13:59:14.501+00:00 readOnce                                   topics="libp2p mplexchannel custom" tid=1 s=16U*uBBR7j:677fd62fe0c5bd152c675e42:677fd62ff7548faf70a27174 bytes=73 count=32672
+//TRC 2025-01-09 13:59:14.501+00:00 MsgReceived                                topics="codex blockexcnetworkpeer" tid=1 num=7 count=32673
+
+                // read to received!???
+
+                // run in cluster, same effect???
+                // run native, same effect?
+
+                if (line.Message == "MsgSending")
+                {
+                    // the next line is lpc-write-fast, then chronoswrite
+                    if (i + 2  < clines.Count)
+                    {
+                        var next = clines[i + 2];
+                        if (next.Message == "chronoswrite")
+                        {
+                            // got ya!
+                            gaps.Add(new Gap(line, next));
+                        }
+                        else
+                        {
+                            var aaaa = "what is it?!";
+                        }
+                    }
+                }
+            }
+
+            gaps = gaps.OrderByDescending(g => g.GapSpan.TotalMilliseconds).ToList();
+
+            var iiii = 0;
+
+        }
+
+        public class Gap
+        {
+            public Gap(CodexLogLine line, CodexLogLine next)
+            {
+                Line = line;
+                Next = next;
+            }
+
+            public CodexLogLine Line { get; }
+            public CodexLogLine Next { get; }
+
+            public TimeSpan GapSpan
+            {
+                get
+                {
+                    return Next.TimestampUtc - Line.TimestampUtc;
+                }
+            }
+
+            public override string ToString()
+            {
+                return $"[{GapSpan.TotalMilliseconds} ms]";
+            }
+        }
+
+        private void ProcessTimes(CodexLogLine cline)
+        {
+            // reqCreatedTime
+            // wantHaveSentTimes
+            // presenceRecvTimes
+            // wantBlkSentTimes
+            // blkRecvTimes
+            // cancelSentTimes
+            // resolveTimes
+
+        }
+
+        public class BlockReqTimes
+        {
+            public TimeSpan CreateToWantHaveSent { get; set; }
 
-            PerformTwoClientTest(uploader, downloader);
         }
 
         private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
         {
-            PerformTwoClientTest(uploader, downloader, 10.MB());
+            PerformTwoClientTest(uploader, downloader, 100.MB());
         }
 
         private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
@@ -51,11 +196,12 @@ namespace CodexReleaseTests.DataTests
             var contentId = uploader.UploadFile(testFile);
             AssertNodesContainFile(contentId, uploader);
 
-            var downloadedFile = downloader.DownloadContent(contentId);
+            var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
             AssertNodesContainFile(contentId, uploader, downloader);
 
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(15.0)), "Too slow!");
+
             testFile.AssertIsEqual(downloadedFile);
-            CheckLogForErrors(uploader, downloader);
         }
     }
 }

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -1,20 +1,111 @@
-using CodexTests;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Utils;
 
 namespace CodexReleaseTests.MarketTests
 {
-    public class ContractFailedTest : CodexDistTest
+    public class ContractFailedTest : MarketplaceAutoBootstrapDistTest
     {
+        protected override int NumberOfHosts => 4;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => 1.GB();
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(1.0);
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
         [Test]
-        [Ignore("TODO - Test in which hosts are punished for failing a contract")]
         public void ContractFailed()
         {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+            StartValidator();
 
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            hosts.BringOffline(waitTillStopped: true);
+
+            WaitForSlotFreedEvents();
+
+            request.WaitForContractFailed();
+        }
+
+        private void WaitForSlotFreedEvents()
+        {
+            Log(nameof(WaitForSlotFreedEvents));
+
+            var start = DateTime.UtcNow;
+            var timeout = CalculateContractFailTimespan();
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var slotFreed = events.GetSlotFreedEvents();
+                if (slotFreed.Length == NumberOfHosts)
+                {
+                    Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
+                    return;
+                }
+                GetContracts().WaitUntilNextPeriod();
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private TimeSpan CalculateContractFailTimespan()
+        {
+            var config = GetContracts().Deployment.Config;
+            var maxSlashesBeforeSlotFreed = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
+            var numProofsMissedBeforeSlash = Convert.ToInt32(config.Collateral.SlashCriterion);
+
+            var periodDuration = GetPeriodDuration();
+            var requiredNumMissedProofs = maxSlashesBeforeSlotFreed * numProofsMissedBeforeSlash;
+
+            // Each host could miss 1 proof per period,
+            // so the time we should wait is period time * requiredNum of missed proofs.
+            // Except: the proof requirement has a concept of "downtime":
+            // a segment of time where proof is not required.
+            // We calculate the probability of downtime and extend the waiting
+            // timeframe by a factor, such that all hosts are highly likely to have 
+            // failed a sufficient number of proofs.
+
+            float n = requiredNumMissedProofs;
+            return periodDuration * n * GetDowntimeFactor(config);
+        }
+
+        private float GetDowntimeFactor(MarketplaceConfig config)
+        {
+            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
+            float downtime = numBlocksInDowntimeSegment;
+            float window = 256.0f;
+            var chanceOfDowntime = downtime / window;
+            return 1.0f + chanceOfDowntime + chanceOfDowntime;
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(5.MB()));
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = TimeSpan.FromHours(1.0),
+                Expiry = TimeSpan.FromHours(0.2),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 1, // Require a proof every period
+                RequiredCollateral = 1.Tst()
+            });
         }
     }
 }

Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

@@ -11,7 +11,7 @@ namespace CodexReleaseTests.MarketTests
     {
         private const int FilesizeMb = 10;
 
-        protected override int NumberOfHosts => 4;
+        protected override int NumberOfHosts => 6;
         protected override int NumberOfClients => 1;
         protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
         protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
@@ -46,8 +46,11 @@ namespace CodexReleaseTests.MarketTests
             {
                 Duration = GetContractDuration(),
                 Expiry = GetContractExpiry(),
-                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
-                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                // TODO: this should work with NumberOfHosts, but
+                // an ongoing issue makes hosts sometimes not pick up slots.
+                // When it's resolved, we can reduce the number of hosts and slim down this test.
+                MinRequiredNumberOfNodes = 3,
+                NodeFailureTolerance = 1,
                 PricePerSlotPerSecond = pricePerSlotPerSecond,
                 ProofProbability = 20,
                 RequiredCollateral = 1.Tst()
@@ -66,8 +69,7 @@ namespace CodexReleaseTests.MarketTests
 
         private TimeSpan Get8TimesConfiguredPeriodDuration()
         {
-            var config = GetContracts().Deployment.Config;
-            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+            return GetPeriodDuration() * 8.0;
         }
     }
 }

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -19,7 +19,7 @@ namespace CodexReleaseTests.MarketTests
         protected override void LifecycleStart(TestLifecycle lifecycle)
         {
             base.LifecycleStart(lifecycle);
-            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var geth = StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
             handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
         }
@@ -40,6 +40,12 @@ namespace CodexReleaseTests.MarketTests
             return handles[Get()].Contracts;
         }
 
+        protected TimeSpan GetPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period));
+        }
+
         protected abstract int NumberOfHosts { get; }
         protected abstract int NumberOfClients { get; }
         protected abstract ByteSize HostAvailabilitySize { get; }
@@ -101,6 +107,17 @@ namespace CodexReleaseTests.MarketTests
             );
         }
 
+        public ICodexNode StartValidator()
+        {
+            return StartCodex(s => s
+                .WithName("validator")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsValidator()
+                )
+            );
+        }
+
         public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
         {
             var fills = GetOnChainSlotFills(possibleHosts);
@@ -177,9 +194,17 @@ namespace CodexReleaseTests.MarketTests
 
         private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract)
         {
-            var events = GetContracts().GetEvents(GetTestRunTimeRange());
-            var submitEvent = events.GetStorageRequests().Single(e => e.RequestId.ToHex(false) == contract.PurchaseId);
-            return submitEvent.Block.Utc;
+            return Time.Retry<DateTime>(() =>
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var submitEvent = events.GetStorageRequests().SingleOrDefault(e => e.RequestId.ToHex(false) == contract.PurchaseId);
+                if (submitEvent == null)
+                {
+                    // We're too early.
+                    throw new TimeoutException(nameof(GetContractOnChainSubmittedUtc) + "StorageRequest not found on-chain.");
+                }
+                return submitEvent.Block.Utc;
+            }, nameof(GetContractOnChainSubmittedUtc));
         }
 
         private TestToken GetContractCostPerSlot(TestToken pricePerSlotPerSecond, TimeSpan slotDuration)

Tests/DistTestCore/NameUtils.cs

@@ -55,12 +55,23 @@ namespace DistTestCore
         private static string FormatArguments(TestContext.TestAdapter test)
         {
             if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
+            return $"[{string.Join(',', test.Arguments.Select(FormatArgument).ToArray())}]";
+        }
+        
+        private static string FormatArgument(object? obj)
+        {
+            if (obj == null) return "";
+            var str = obj.ToString();
+            if (string.IsNullOrEmpty(str)) return "";
+            return ReplaceInvalidCharacters(str);
         }
 
         private static string ReplaceInvalidCharacters(string name)
         {
-            return name.Replace(":", "_");
+            return name
+                .Replace(":", "_")
+                .Replace("/", "_")
+                .Replace("\\", "_");
         }
 
         private static string DetermineFolder(LogConfig config, DateTime start)

Tests/ExperimentalTests/BasicTests/ExampleTests.cs

@@ -53,9 +53,9 @@ namespace CodexTests.BasicTests
         [Test]
         public void GethBootstrapTest()
         {
-            var boot = Ci.StartGethNode(s => s.WithName("boot").IsMiner());
-            var disconnected = Ci.StartGethNode(s => s.WithName("disconnected"));
-            var follow = Ci.StartGethNode(s => s.WithBootstrapNode(boot).WithName("follow"));
+            var boot = StartGethNode(s => s.WithName("boot").IsMiner());
+            var disconnected = StartGethNode(s => s.WithName("disconnected"));
+            var follow = StartGethNode(s => s.WithBootstrapNode(boot).WithName("follow"));
 
             Thread.Sleep(12000);
 

Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs

@@ -28,7 +28,7 @@ namespace CodexTests.BasicTests
                 plusSizeBytes
             );
 
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
             
             var numberOfHosts = 5;

Tests/ExperimentalTests/CodexDistTest.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
 using CodexNetDeployer;
 using CodexPlugin;
 using CodexPlugin.OverwatchSupport;
@@ -7,18 +8,21 @@ using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
 using DistTestCore.Logs;
+using GethPlugin;
 using Logging;
 using MetricsPlugin;
 using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using OverwatchTranscript;
+using Utils;
 
 namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {
         private static readonly Dictionary<TestLifecycle, CodexTranscriptWriter> writers = new Dictionary<TestLifecycle, CodexTranscriptWriter>();
+        private static readonly Dictionary<TestLifecycle, BlockCache> blockCaches = new Dictionary<TestLifecycle, BlockCache>();
 
         public CodexDistTest()
         {
@@ -73,6 +77,11 @@ namespace CodexTests
             return group;
         }
 
+        public IGethNode StartGethNode(Action<IGethSetup> setup)
+        {
+            return Ci.StartGethNode(GetBlockCache(), setup);
+        }
+
         public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
         {
             return new PeerConnectionTestHelpers(GetTestLog());
@@ -147,15 +156,6 @@ namespace CodexTests
                 node.Space().ToString() + Environment.NewLine;
         }
 
-        // Disabled for now: Makes huge log files!
-        //private string GetNodeMetrics(IMetricsAccess? metrics)
-        //{
-        //    if (metrics == null) return "No metrics enabled";
-        //    var m = metrics.GetAllMetrics();
-        //    if (m == null) return "No metrics received";
-        //    return m.AsCsv();
-        //}
-
         protected virtual void OnCodexSetup(ICodexSetup setup)
         {
         }
@@ -223,6 +223,16 @@ namespace CodexTests
             if (!outputFile.EndsWith(".owts")) outputFile += ".owts";
             return outputFile;
         }
+
+        private BlockCache GetBlockCache()
+        {
+            var lifecycle = Get();
+            if (!blockCaches.ContainsKey(lifecycle))
+            {
+                blockCaches[lifecycle] = new BlockCache();
+            }
+            return blockCaches[lifecycle];
+        }
     }
 
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]

Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -20,7 +20,7 @@ namespace CodexTests.DownloadConnectivityTests
         [Test]
         public void MarketplaceDoesNotInterfereWithPeerDownload()
         {
-            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var geth = StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
             var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TstWei())));

Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -29,7 +29,7 @@ namespace CodexTests.PeerDiscoveryTests
         [Test]
         public void MarketplaceDoesNotInterfereWithPeerDiscovery()
         {
-            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var geth = StartGethNode(s => s.IsMiner());
             var contracts = Ci.StartCodexContracts(geth);
             var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m
                 .WithInitial(10.Eth(), 1000.TstWei())));

Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs

@@ -30,7 +30,7 @@ namespace CodexTests.UtilityTests
         [Ignore("Used to debug testnet bots.")]
         public void BotRewardTest()
         {
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
+            var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
             var gethInfo = CreateGethInfo(geth, contracts);
 

Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs

@@ -1,7 +1,7 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Moq;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using NUnit.Framework;
 
 namespace FrameworkTests.NethereumWorkflow

Tools/AutoClient/App.cs

@@ -1,4 +1,5 @@
-using Logging;
+using AutoClient.Modes.FolderStore;
+using Logging;
 
 namespace AutoClient
 {
@@ -19,6 +20,15 @@ namespace AutoClient
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
+
+            if (!string.IsNullOrEmpty(config.FolderToStore))
+            {
+                FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
+            }
+            else
+            {
+                FolderWorkDispatcher = null!;
+            }
         }
 
         public Configuration Config { get; }
@@ -27,6 +37,7 @@ namespace AutoClient
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public CidRepo CidRepo { get; }
         public Performance Performance { get; }
+        public FolderWorkDispatcher FolderWorkDispatcher { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -0,0 +1,165 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class AutomaticPurchaser
+    {
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly CodexNode codex;
+        private Task workerTask = Task.CompletedTask;
+        private App app => instance.App;
+
+        public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
+        {
+            this.log = log;
+            this.instance = instance;
+            this.codex = codex;
+        }
+
+        public void Start()
+        {
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
+        }
+
+        private async Task Worker()
+        {
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(instance.NodeId);
+            if (cid == null) return;
+
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+            await codex.DownloadCid(filename, cid, size);
+
+            DeleteFile(filename);
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            try
+            {
+                var cid = await codex.UploadFile(file);
+                var response = await codex.RequestStorage(cid);
+                return response.PurchaseId;
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (!app.Cts.Token.IsCancellationRequested)
+                {
+                    var purchase = await codex.GetStoragePurchase(pid);
+                    if (purchase == null)
+                    {
+                        await FixedShortDelay();
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                        continue;
+                    }
+                    if (purchase.IsCancelled)
+                    {
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (purchase.IsError)
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (purchase.IsFinished)
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (purchase.IsStarted)
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, app.Cts.Token);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -17,6 +17,7 @@
             lock (_lock)
             {
                 entries.Add(new CidEntry(nodeId, cid, knownSize));
+                if (entries.Count > 1000) entries.Clear();
             }
         }
 

Tools/AutoClient/CodexInstance.cs

@@ -0,0 +1,161 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Nethereum.Model;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public interface ICodexInstance
+    {
+        string NodeId { get; }
+        App App { get; }
+        CodexApi Codex { get; }
+        HttpClient Client { get; }
+        Address Address { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            App = app;
+            Codex = codex;
+            Client = client;
+            Address = address;
+            NodeId = Guid.NewGuid().ToString();
+        }
+
+        public string NodeId { get; }
+        public App App { get; }
+        public CodexApi Codex { get; }
+        public HttpClient Client { get; }
+        public Address Address { get; }
+    }
+
+    public class CodexNode
+    {
+        private readonly App app;
+        private readonly ICodexInstance codex;
+
+        public CodexNode(App app, ICodexInstance instance)
+        {
+            this.app = app;
+            codex = instance;
+        }
+
+        public async Task DownloadCid(string filename, string cid, long? size)
+        {
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                using var fileStream = File.OpenWrite(filename);
+                var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
+                fileResponse.Stream.CopyTo(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.DownloadSuccessful(size, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
+            }
+        }
+
+        public async Task<ContentId> UploadFile(string filename)
+        {
+            using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream, filename);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
+
+        public async Task<RequestStorageResult> RequestStorage(ContentId cid)
+        {
+            app.Log.Debug("Requesting storage for " + cid.Id);
+            var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
+
+            app.Log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
+
+            return new RequestStorageResult(result, new ContentId(encoded));
+        }
+
+        public class RequestStorageResult
+        {
+            public RequestStorageResult(string purchaseId, ContentId encodedCid)
+            {
+                PurchaseId = purchaseId;
+                EncodedCid = encodedCid;
+            }
+
+            public string PurchaseId { get; }
+            public ContentId EncodedCid { get; }
+
+            public override string ToString()
+            {
+                return $"{PurchaseId} (cid: {EncodedCid})";
+            }
+        }
+
+        public async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
+        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
+        {
+            app.Log.Debug($"Uploading file...");
+            var response = await codex.Codex.UploadAsync(
+                content_type: "application/octet-stream",
+                content_disposition: $"attachment; filename=\"{filename}\"",
+                fileStream, app.Cts.Token);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> GetEncodedCid(string pid)
+        {
+            try
+            {
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
+            }
+            catch (Exception ex)
+            {
+                app.Log.Error(ex.ToString());
+                throw;
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -5,7 +5,8 @@ namespace AutoClient
     public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
-        public string CodexEndpoints { get; set; } = "http://localhost:8080";
+        public string CodexEndpoints { get; set; } =
+            "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -14,19 +15,22 @@ namespace AutoClient
         public int NumConcurrentPurchases { get; set; } = 10;
 
         [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
-        public int ContractDurationMinutes { get; set; } = 60 * 6;
+        public int ContractDurationMinutes { get; set; } =
+            60 * 24 * 6; // 6 days
+            //60 * 6; 6 hours
+        // Cluster nodes configured for max 7-day storage.
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 15;
+        public int ContractExpiryMinutes { get; set; } = 60;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
-        public int NumHosts { get; set; } = 10;
+        public int NumHosts { get; set; } = 5;
 
         [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
-        public int HostTolerance { get; set; } = 5;
+        public int HostTolerance { get; set; } = 1;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
-        public int Price { get; set; } = 10;
+        public int Price { get; set; } = 1000;
 
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
@@ -34,6 +38,9 @@ namespace AutoClient
         [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
         public int FileSizeMb { get; set; } = 0;
 
+        [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
+        public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
+
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -0,0 +1,58 @@
+using static AutoClient.Modes.FolderStore.FileWorker;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileStatus : JsonBacked<WorkerStatus>
+    {
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
+            : base(app, folder, filePath + ".json")
+        {
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public bool IsBusy()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                !p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyRunning()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyFailed()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            var mostRecent = GetMostRecent();
+            if (mostRecent == null) return false;
+
+            return mostRecent.Expiry.HasValue;
+        }
+
+        protected WorkerPurchase? GetMostRecent()
+        {
+            if (!State.Purchases.Any()) return null;
+            var maxCreated = State.Purchases.Max(p => p.Created);
+            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -0,0 +1,293 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileWorker : FileStatus
+    {
+        private readonly App app;
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly string sourceFilename;
+        private readonly Action onFileUploaded;
+        private readonly Action onNewPurchase;
+        private readonly CodexNode codex;
+
+        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
+            : base(app, folder, fileIndex.File + ".json", purchaseInfo)
+        {
+            this.app = app;
+            log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
+            this.instance = instance;
+            this.purchaseInfo = purchaseInfo;
+            sourceFilename = fileIndex.File;
+            if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
+            this.onFileUploaded = onFileUploaded;
+            this.onNewPurchase = onNewPurchase;
+            codex = new CodexNode(app, instance);
+        }
+
+        public int FailureCounter => State.FailureCounter;
+
+        protected override void OnNewState(WorkerStatus newState)
+        {
+            newState.LastUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update()
+        {
+            try
+            {
+                if (IsCurrentlyRunning() && UpdatedRecently()) return;
+
+                Log($"Updating for '{sourceFilename}'...");
+                await EnsureRecentPurchase();
+                SaveState();
+                app.Log.Log("");
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Exception during fileworker update: " + exc);
+                State.Error = exc.ToString();
+                SaveState();
+                throw;
+            }
+        }
+
+        private bool UpdatedRecently()
+        {
+            var now = DateTime.UtcNow;
+            return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
+        }
+
+        private async Task<string> EnsureCid()
+        {
+            Log($"Checking CID...");
+
+            if (!string.IsNullOrEmpty(State.EncodedCid) &&
+                await DoesCidExistInNetwork(State.EncodedCid))
+            {
+                Log("Encoded-CID successfully found in the network.");
+                // TODO: Using the encoded CID currently would result in double-encoding of the dataset.
+                // See: https://github.com/codex-storage/nim-codex/issues/1005
+                // Always use the basic CID for now, even though we have to repeat the encoding.
+                // When using encoded CID works: return State.EncodedCid;
+            }
+
+            if (!string.IsNullOrEmpty(State.Cid) &&
+                await DoesCidExistInNetwork(State.Cid))
+            {
+                Log("Basic-CID successfully found in the network.");
+                return State.Cid;
+            }
+
+            if (string.IsNullOrEmpty(State.Cid))
+            {
+                Log("File was not previously uploaded.");
+            }
+
+            Log($"Uploading...");
+            var cid = await codex.UploadFile(sourceFilename);
+            onFileUploaded();
+            Log("Got Basic-CID: " + cid);
+            State.Cid = cid.Id;
+            SaveState();
+            return State.Cid;
+        }
+
+        private async Task<bool> DoesCidExistInNetwork(string cid)
+        {
+            try
+            {
+                // This should not take longer than a few seconds. If it does, cancel it.
+                var cts = new CancellationTokenSource();
+                var cancelTask = Task.Run(() =>
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(15));
+                    cts.Cancel();
+                });
+
+                var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
+                if (manifest == null) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+        private async Task EnsureRecentPurchase()
+        {
+            Log($"Checking recent purchase...");
+            var recent = GetMostRecent();
+            if (recent == null)
+            {
+                Log($"No recent purchase.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            await UpdatePurchase(recent);
+
+            if (recent.Expiry.HasValue)
+            {
+                Log($"Purchase has failed or expired.");
+                await MakeNewPurchase();
+                State.FailureCounter++;
+                return;
+            }
+
+            if (recent.Finish.HasValue)
+            {
+                Log($"Purchase has finished.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
+            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
+            {
+                Log($"Purchase is going to expire soon.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            if (!recent.Submitted.HasValue)
+            {
+                Log($"Purchase is waiting to be submitted.");
+                return;
+            }
+
+            if (recent.Submitted.HasValue && !recent.Started.HasValue)
+            {
+                Log($"Purchase is submitted and waiting to start.");
+                return;
+            }
+
+            Log($"Purchase is running.");
+        }
+
+        private async Task UpdatePurchase(WorkerPurchase recent)
+        {
+            if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
+            var now = DateTime.UtcNow;
+
+            var purchase = await codex.GetStoragePurchase(recent.Pid);
+            if (purchase == null)
+            {
+                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
+                recent.Expiry = now;
+                return;
+            }
+
+            if (purchase.IsSubmitted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+            }
+            if (purchase.IsStarted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+            }
+            if (purchase.IsCancelled)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsError)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsFinished)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+                if (!recent.Finish.HasValue) recent.Finish = now;
+            }
+            State.LastUpdate = now;
+            SaveState();
+        }
+
+        private async Task MakeNewPurchase()
+        {
+            var cid = await EnsureCid();
+            if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
+
+            Log($"Creating new purchase...");
+            var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
+            var purchaseId = response.PurchaseId;
+            var encodedCid = response.EncodedCid;
+            if (string.IsNullOrEmpty(purchaseId) ||
+                purchaseId == "Unable to encode manifest" ||
+                purchaseId == "Purchasing not available" ||
+                purchaseId == "Expiry required" ||
+                purchaseId == "Expiry needs to be in future" ||
+                purchaseId == "Expiry has to be before the request's end (now + duration)")
+            {
+                throw new InvalidOperationException(purchaseId);
+            }
+
+            var newPurchase = new WorkerPurchase
+            {
+                Created = DateTime.UtcNow,
+                Pid = purchaseId
+            };
+            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
+            State.EncodedCid = encodedCid.Id;
+            SaveState();
+            onNewPurchase();
+
+            Log($"New purchase created. PID: '{purchaseId}'.");
+            Log("Got Encoded-CID: " + encodedCid);
+            Log("Waiting for submit...");
+            Thread.Sleep(500);
+
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (DateTime.UtcNow < timeout)
+            {
+                Thread.Sleep(5000);
+                await UpdatePurchase(newPurchase);
+                if (newPurchase.Submitted.HasValue)
+                {
+                    Log("New purchase successfully submitted.");
+                    return;
+                }
+            }
+            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
+        private string GetFileTag(FileIndex filename)
+        {
+            return $"({filename.Index.ToString("00000")}) ";
+        }
+
+        [Serializable]
+        public class WorkerStatus
+        {
+            public DateTime LastUpdate { get; set; }
+            public string Cid { get; set; } = string.Empty;
+            public string EncodedCid { get; set; } = string.Empty;
+            public int FailureCounter { get; set; } = 0;
+            public string Error { get; set; } = string.Empty;
+            public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
+        }
+
+        [Serializable]
+        public class WorkerPurchase
+        {
+            public string Pid { get; set; } = string.Empty;
+            public DateTime Created { get; set; }
+            public DateTime? Submitted { get; set; }
+            public DateTime? Started { get; set; }
+            public DateTime? Expiry { get; set; }
+            public DateTime? Finish { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -0,0 +1,70 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkDispatcher
+    {
+        private readonly string[] files = Array.Empty<string>();
+        private readonly ILog log;
+        private int index = 0;
+        private int busyCount = 0;
+
+        public FolderWorkDispatcher(ILog log, string folder)
+        {
+            var fs = Directory.GetFiles(folder);
+            var result = new List<string>();
+            foreach (var f in fs)
+            {
+                if (!f.ToLowerInvariant().Contains(".json"))
+                {
+                    var info = new FileInfo(f);
+                    if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
+                    {
+                        result.Add(f);
+                    }
+                }
+            }
+            files = result.ToArray();
+            this.log = log;
+        }
+
+        public FileIndex GetFileToCheck()
+        {
+            if (busyCount > 0)
+            {
+                log.Log("");
+                log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
+                log.Log("");
+                ResetIndex();
+                Thread.Sleep(TimeSpan.FromMinutes(1));
+            }
+
+            var file = new FileIndex(files[index], index);
+            index = (index + 1) % files.Length;
+            return file;
+        }
+
+        public void ResetIndex()
+        {
+            index = 0;
+            busyCount = 0;
+        }
+
+        public void WorkerIsBusy()
+        {
+            busyCount++;
+        }
+    }
+
+    public class FileIndex
+    {
+        public FileIndex(string file, int index)
+        {
+            File = file;
+            Index = index;
+        }
+
+        public string File { get; }
+        public int Index { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -0,0 +1,136 @@
+using CodexOpenApi;
+using System.IO.Compression;
+using static AutoClient.Modes.FolderStore.FolderWorkOverview;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
+    {
+        private const string OverviewFilename = "codex_folder_saver_overview.json";
+        private readonly App app;
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
+            : base(app, folder, Path.Combine(folder, OverviewFilename))
+        {
+            this.app = app;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        protected override void OnNewState(WorkMonitorStatus newState)
+        {
+            newState.LastOverviewUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update(ICodexInstance instance)
+        {
+            var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
+
+            var total = 0;
+            var successful = 0;
+            var failed = 0;
+            foreach (var file in jsonFiles)
+            {
+                try
+                {
+                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
+                    total++;
+                    if (worker.IsCurrentlyRunning()) successful++;
+                    if (worker.IsCurrentlyFailed()) failed++;
+                }
+                catch (Exception exc)
+                {
+                    app.Log.Error("Exception in workoverview update: " + exc);
+                }
+            }
+
+            State.TotalFiles = total;
+            State.SuccessfulStored = successful;
+            State.StoreFailed = failed;
+            SaveState();
+
+            if (State.UncommitedChanges > 3)
+            {
+                State.UncommitedChanges = 0;
+                SaveState();
+
+                await CreateNewOverviewZip(jsonFiles, FilePath, instance);
+            }
+        }
+
+        public void MarkUncommitedChange()
+        {
+            State.UncommitedChanges++;
+            SaveState();
+        }
+
+        private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
+        {
+            Log("");
+            Log("");
+            Log("Creating new overview zipfile...");
+            var zipFilename = CreateZipFile(jsonFiles, filePath);
+
+            Log("Uploading to Codex...");
+            try
+            {
+                var codex = new CodexNode(app, instance);
+                var cid = await codex.UploadFile(zipFilename);
+                Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
+                Log("Requesting storage for it...");
+                var result = await codex.RequestStorage(cid);
+                Log("Storage requested. Purchase ID: " + result);
+
+                var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
+                File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
+                Log($">>> [{outFile}] has been updated. <<<");
+            }
+            catch (Exception exc)
+            {
+                Log("Failed to upload new overview zipfile: " + exc);
+            }
+            Log("");
+            Log("");
+        }
+
+        private string CreateZipFile(List<string> jsonFiles, string filePath)
+        {
+            var zipFilename = Guid.NewGuid().ToString() + ".zip";
+
+            using (var memoryStream = new MemoryStream())
+            {
+                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
+                {
+                    archive.CreateEntryFromFile(filePath, "overview.json");
+                    foreach (var file in jsonFiles)
+                    {
+                        archive.CreateEntryFromFile(file, Path.GetFileName(file));
+                    }
+                }
+
+                using (var fileStream = new FileStream(zipFilename, FileMode.Create))
+                {
+                    memoryStream.Seek(0, SeekOrigin.Begin);
+                    memoryStream.CopyTo(fileStream);
+                }
+            }
+            return zipFilename;
+        }
+
+        private void Log(string msg)
+        {
+            app.Log.Log(msg);
+        }
+
+        [Serializable]
+        public class WorkMonitorStatus
+        {
+            public int TotalFiles { get; set; }
+            public int SuccessfulStored { get; set; }
+            public int StoreFailed { get; set; }
+
+            public DateTime LastOverviewUpdate { get; set; }
+            public int UncommitedChanges { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/JsonBacked.cs

@@ -0,0 +1,58 @@
+using Newtonsoft.Json;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public abstract class JsonBacked<T> where T : new()
+    {
+        private readonly App app;
+
+        protected JsonBacked(App app, string folder, string filePath)
+        {
+            this.app = app;
+            Folder = folder;
+            FilePath = filePath;
+            LoadState();
+        }
+
+        private void LoadState()
+        {
+            try
+            {
+                if (!File.Exists(FilePath))
+                {
+                    State = new T();
+                    OnNewState(State);
+                    SaveState();
+                }
+                var text = File.ReadAllText(FilePath);
+                State = JsonConvert.DeserializeObject<T>(text)!;
+                if (State == null) throw new Exception("Didn't deserialize " + FilePath);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to load state: " + exc);
+            }
+        }
+
+        protected string Folder { get; }
+        protected string FilePath { get; }
+        protected T State { get; private set; } = default!;
+
+        protected virtual void OnNewState(T newState)
+        {
+        }
+
+        protected void SaveState()
+        {
+            try
+            {
+                var json = JsonConvert.SerializeObject(State);
+                File.WriteAllText(FilePath, json);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to save state: " + exc);
+            }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/PurchaseInfo.cs

@@ -0,0 +1,18 @@
+namespace AutoClient.Modes.FolderStore
+{
+    public class PurchaseInfo
+    {
+        public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
+        {
+            PurchaseDurationTotal = purchaseDurationTotal;
+            PurchaseDurationSafe = purchaseDurationSafe;
+
+            if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
+            if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
+            if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
+        }
+
+        public TimeSpan PurchaseDurationTotal { get; }
+        public TimeSpan PurchaseDurationSafe { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -0,0 +1,86 @@
+using AutoClient.Modes.FolderStore;
+
+namespace AutoClient.Modes
+{
+    public class FolderStoreMode : IMode
+    {
+        private readonly App app;
+        private readonly string folder;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly CancellationTokenSource cts = new CancellationTokenSource();
+        private Task checkTask = Task.CompletedTask;
+
+        public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
+        {
+            this.app = app;
+            this.folder = folder;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public void Start(ICodexInstance instance, int index)
+        {
+            checkTask = Task.Run(async () =>
+            {
+                try
+                {
+                    await RunChecker(instance);
+                }
+                catch (Exception ex)
+                {
+                    app.Log.Error("Exception in FolderStoreMode worker: " + ex);
+                    Environment.Exit(1);
+                }
+            });
+        }
+
+        private async Task RunChecker(ICodexInstance instance)
+        {
+            var i = 0;
+            while (!cts.IsCancellationRequested)
+            {
+                Thread.Sleep(2000);
+
+                var worker = await ProcessWorkItem(instance);
+                if (worker.FailureCounter > 5)
+                {
+                    throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
+                }
+                i++;
+
+                if (i > 5)
+                {
+                    i = 0;
+                    var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+                    await overview.Update(instance);
+                }
+            }
+        }
+
+        private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
+        {
+            var file = app.FolderWorkDispatcher.GetFileToCheck();
+            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
+            await worker.Update();
+            if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
+            return worker;
+        }
+
+        private void OnFileUploaded()
+        {
+        }
+
+        private void OnNewPurchase()
+        {
+            app.FolderWorkDispatcher.ResetIndex();
+
+            var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+            overview.MarkUncommitedChange();
+        }
+
+        public void Stop()
+        {
+            cts.Cancel();
+            checkTask.Wait();
+        }
+    }
+}

Tools/AutoClient/Modes/Mode.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AutoClient.Modes
+{
+    public interface IMode
+    {
+        void Start(ICodexInstance instance, int index);
+        void Stop();
+    }
+}

Tools/AutoClient/Modes/PurchasingMode.cs

@@ -1,32 +1,23 @@
-using CodexOpenApi;
-using Logging;
-using Utils;
+using Logging;
 
-namespace AutoClient
+namespace AutoClient.Modes
 {
-    public class CodexUser
+    public class PurchasingMode : IMode
     {
+        private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
         private readonly App app;
-        private readonly CodexApi codex;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly List<Purchaser> purchasers = new List<Purchaser>();
         private Task starterTask = Task.CompletedTask;
-        private readonly string nodeId = Guid.NewGuid().ToString();
 
-        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        public PurchasingMode(App app)
         {
             this.app = app;
-            this.codex = codex;
-            this.client = client;
-            this.address = address;
         }
 
-        public void Start(int index)
+        public void Start(ICodexInstance instance, int index)
         {
             for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
             {
-                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+                purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
             }
 
             var delayPerPurchaser =

Tools/AutoClient/Performance.cs

@@ -16,11 +16,13 @@ namespace AutoClient
             Log($"Download failed: {ex}");
         }
 
-        public void DownloadSuccessful(long size, TimeSpan time)
+        public void DownloadSuccessful(long? size, TimeSpan time)
         {
+            if (!size.HasValue) return;
+
             long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
             if (milliseconds < 1) milliseconds = 1;
-            long bytesPerSecond = 1000 * (size / milliseconds);
+            long bytesPerSecond = 1000 * (size.Value / milliseconds);
             Log($"Download successful: {bytesPerSecond} bytes per second");
         }
 

Tools/AutoClient/Program.cs

@@ -1,11 +1,14 @@
 using ArgsUniform;
 using AutoClient;
+using AutoClient.Modes;
+using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;
 
 public class Program
 {
     private readonly App app;
+    private readonly List<IMode> modes = new List<IMode>();
 
     public Program(Configuration config)
     {
@@ -31,36 +34,60 @@ public class Program
 
     public async Task Run()
     {
-        var codexUsers = await CreateUsers();
+        var codexInstances = await CreateCodexInstances();
 
         var i = 0;
-        foreach (var user in codexUsers)
+        foreach (var cdx in codexInstances)
         {
-            user.Start(i);
+            var mode = CreateMode();
+            modes.Add(mode);
+
+            mode.Start(cdx, i);
             i++;
         }
 
         app.Cts.Token.WaitHandle.WaitOne();
 
-        foreach (var user in codexUsers) user.Stop();
+        foreach (var mode in modes) mode.Stop();
+        modes.Clear();
 
         app.Log.Log("Done");
     }
 
-    private async Task<CodexUser[]> CreateUsers()
+    private IMode CreateMode()
+    {
+        if (!string.IsNullOrEmpty(app.Config.FolderToStore))
+        {
+            return CreateFolderStoreMode();
+        }
+
+        return new PurchasingMode(app);
+    }
+
+    private IMode CreateFolderStoreMode()
+    {
+        if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
+
+        return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
+            purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
+            purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
+        ));
+    }
+
+    private async Task<CodexInstance[]> CreateCodexInstances()
     {
         var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
-        var result = new List<CodexUser>();
+        var result = new List<CodexInstance>();
 
         foreach (var e in endpointStrs)
         {
-            result.Add(await CreateUser(e));
+            result.Add(await CreateCodexInstance(e));
         }
 
         return result.ToArray();
     }
 
-    private async Task<CodexUser> CreateUser(string endpoint)
+    private async Task<CodexInstance> CreateCodexInstance(string endpoint)
     {
         var splitIndex = endpoint.LastIndexOf(':');
         var host = endpoint.Substring(0, splitIndex);
@@ -72,6 +99,7 @@ public class Program
         );
 
         var client = new HttpClient();
+        client.Timeout = TimeSpan.FromMinutes(60.0);
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
@@ -79,7 +107,7 @@ public class Program
         await CheckCodex(codex);
         app.Log.Log("OK");
 
-        return new CodexUser(
+        return new CodexInstance(
             app,
             codex,
             client,
@@ -105,4 +133,4 @@ public class Program
     {
         Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
     }
-}
+}

Tools/AutoClient/Purchaser.cs

@@ -1,261 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public class Purchaser
-    {
-        private readonly App app;
-        private readonly string nodeId;
-        private readonly ILog log;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly CodexApi codex;
-        private Task workerTask = Task.CompletedTask;
-
-        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
-        {
-            this.app = app;
-            this.nodeId = nodeId;
-            this.log = log;
-            this.client = client;
-            this.address = address;
-            this.codex = codex;
-        }
-
-        public void Start()
-        {
-            workerTask = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            workerTask.Wait();
-        }
-
-        private async Task Worker()
-        {
-            log.Log("Worker started.");
-            while (!app.Cts.Token.IsCancellationRequested)
-            {
-                try
-                {
-                    var pid = await StartNewPurchase();
-                    await WaitTillFinished(pid);
-                    await DownloadForeignCid();
-                }
-                catch (Exception ex)
-                {
-                    log.Error("Worker failed with: " + ex);
-                    await Task.Delay(TimeSpan.FromHours(6));
-                }
-            }
-        }
-
-        private async Task DownloadForeignCid()
-        {
-            var cid = app.CidRepo.GetForeignCid(nodeId);
-            if (cid == null) return;
-            var size = app.CidRepo.GetSizeForCid(cid);
-            if (size == null) return;
-
-            try
-            {
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
-                {
-                    using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
-                    fileResponse.Stream.CopyTo(fileStream);
-                }
-                var time = sw.Elapsed;
-                File.Delete(filename);
-                app.Performance.DownloadSuccessful(size.Value, time);
-            }
-            catch (Exception ex)
-            {
-                app.Performance.DownloadFailed(ex);
-            }
-        }
-
-        private async Task<string> StartNewPurchase()
-        {
-            var file = await CreateFile();
-            try
-            {
-                var cid = await UploadFile(file);
-                return await RequestStorage(cid);
-            }
-            finally
-            {
-                DeleteFile(file);
-            }
-        }
-
-        private async Task<string> CreateFile()
-        {
-            return await app.Generator.Generate();
-        }
-
-        private void DeleteFile(string file)
-        {
-            try
-            {
-                File.Delete(file);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error($"Failed to delete file '{file}': {exc}");
-            }
-        }
-
-        private async Task<ContentId> UploadFile(string filename)
-        {
-            using var fileStream = File.OpenRead(filename);
-            try
-            {
-                var info = new FileInfo(filename);
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream, filename);
-                var time = sw.Elapsed;
-                app.Performance.UploadSuccessful(info.Length, time);
-                app.CidRepo.Add(nodeId, cid.Id, info.Length);
-                return cid;
-            }
-            catch (Exception exc)
-            {
-                app.Performance.UploadFailed(exc);
-                throw;
-            }
-        }
-
-        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
-        {
-            log.Debug($"Uploading file...");
-            var response = await codex.UploadAsync(
-                content_type: "application/x-binary",
-                content_disposition: $"attachment; filename=\"{filename}\"",
-                fileStream, app.Cts.Token);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
-
-            log.Debug($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        private async Task<string> RequestStorage(ContentId cid)
-        {
-            log.Debug("Requesting storage for " + cid.Id);
-            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
-            {
-                Collateral = app.Config.RequiredCollateral.ToString(),
-                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = app.Config.NumHosts,
-                Reward = app.Config.Price.ToString(),
-                ProofProbability = "15",
-                Tolerance = app.Config.HostTolerance
-            }, app.Cts.Token);
-
-            log.Debug("Purchase ID: " + result);
-
-            var encoded = await GetEncodedCid(result);
-            app.CidRepo.AddEncoded(cid.Id, encoded);
-
-            return result;
-        }
-
-        private async Task<string> GetEncodedCid(string pid)
-        {
-            try
-            {
-                var sp = (await GetStoragePurchase(pid))!;
-                return sp.Request.Content.Cid;
-            }
-            catch (Exception ex)
-            {
-                log.Error(ex.ToString());
-                throw;
-            }
-        }
-
-        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
-        {
-            // openapi still don't match code.
-            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-            if (string.IsNullOrEmpty(str)) return null;
-            return JsonConvert.DeserializeObject<StoragePurchase>(str);
-        }
-
-        private async Task WaitTillFinished(string pid)
-        {
-            try
-            {
-                var emptyResponseTolerance = 10;
-                while (!app.Cts.Token.IsCancellationRequested)
-                {
-                    var purchase = await GetStoragePurchase(pid);
-                    if (purchase == null)
-                    {
-                        await FixedShortDelay();
-                        emptyResponseTolerance--;
-                        if (emptyResponseTolerance == 0)
-                        {
-                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
-                            await ExpiryTimeDelay();
-                            return;
-                        }
-                        continue;
-                    }
-                    var status = purchase.State.ToLowerInvariant();
-                    if (status.Contains("cancel"))
-                    {
-                        app.Performance.StorageContractCancelled();
-                        return;
-                    }
-                    if (status.Contains("error"))
-                    {
-                        app.Performance.StorageContractErrored(purchase.Error);
-                        return;
-                    }
-                    if (status.Contains("finished"))
-                    {
-                        app.Performance.StorageContractFinished();
-                        return;
-                    }
-                    if (status.Contains("started"))
-                    {
-                        app.Performance.StorageContractStarted();
-                        await FixedDurationDelay();
-                    }
-
-                    await FixedShortDelay();
-                }
-            }
-            catch (Exception ex)
-            {
-                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
-                await ExpiryTimeDelay();
-            }
-        }
-
-        private async Task FixedDurationDelay()
-        {
-            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task ExpiryTimeDelay()
-        {
-            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task FixedShortDelay()
-        {
-            await Task.Delay(15 * 1000, app.Cts.Token);
-        }
-    }
-}

Tools/BiblioTech/Commands/CheckCidCommand.cs

@@ -3,6 +3,8 @@ using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using Discord;
+using BiblioTech.Rewards;
+using System.Data;
 
 namespace BiblioTech.Commands
 {
@@ -68,25 +70,17 @@ namespace BiblioTech.Commands
 
         private async Task<bool> GiveAltruisticRole(CommandContext context, IUser user, string responseMessage)
         {
-            var guildUser = context.Command.User as IGuildUser;
-            if (guildUser != null)
+            try
             {
-                try
-                {
-                    var role = context.Command.Guild.GetRole(Program.Config.AltruisticRoleId);
-                    if (role != null)
-                    {
-                        await guildUser.AddRoleAsync(role);
-                        await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!");
-                        return true;
-                    }
-                }
-                catch (Exception ex)
-                {
-                    await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}");
-                }
+                await Program.RoleDriver.GiveAltruisticRole(user);
+                await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!");
+                return true;
+            }
+            catch (Exception ex)
+            {
+                await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}");
+                return false;
             }
-            return false;
         }
     }
 

Tools/BiblioTech/LoggingRoleDriver.cs

@@ -1,4 +1,5 @@
 using BiblioTech.Rewards;
+using Discord;
 using DiscordRewards;
 using Logging;
 using Newtonsoft.Json;
@@ -14,6 +15,13 @@ namespace BiblioTech
             this.log = log;
         }
 
+        public async Task GiveAltruisticRole(IUser user)
+        {
+            await Task.CompletedTask;
+
+            log.Log($"Give altruistic role to {user.Id}");
+        }
+
         public async Task GiveRewards(GiveRewardsCommand rewards)
         {
             await Task.CompletedTask;

Tools/BiblioTech/Rewards/RewardController.cs

@@ -1,4 +1,5 @@
-using DiscordRewards;
+using Discord;
+using DiscordRewards;
 using Microsoft.AspNetCore.Mvc;
 
 namespace BiblioTech.Rewards
@@ -6,6 +7,7 @@ namespace BiblioTech.Rewards
     public interface IDiscordRoleDriver
     {
         Task GiveRewards(GiveRewardsCommand rewards);
+        Task GiveAltruisticRole(IUser user);
     }
 
     [Route("api/[controller]")]

Tools/BiblioTech/Rewards/RoleDriver.cs

@@ -34,6 +34,18 @@ namespace BiblioTech.Rewards
             await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors);
         }
 
+        public async Task GiveAltruisticRole(IUser user)
+        {
+            var guild = GetGuild();
+            var role = guild.Roles.SingleOrDefault(r => r.Id == Program.Config.AltruisticRoleId);
+            if (role == null) return;
+
+            var guildUser = guild.Users.SingleOrDefault(u => u.Id == user.Id);
+            if (guildUser == null) return;
+
+            await guildUser.AddRoleAsync(role);
+        }
+
         private async Task ProcessRewards(GiveRewardsCommand rewards)
         {
             try

Tools/CodexNetDeployer/Deployer.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
 using CodexDiscordBotPlugin;
 using CodexPlugin;
 using Core;
@@ -59,7 +60,7 @@ namespace CodexNetDeployer
 
             Log("Deploying Geth instance...");
             var gethDeployment = DeployGeth(ci);
-            var gethNode = ci.WrapGethDeployment(gethDeployment);
+            var gethNode = ci.WrapGethDeployment(gethDeployment, new BlockCache());
 
             Log("Geth started. Deploying Codex contracts...");
             var contractsDeployment = ci.DeployCodexContracts(gethNode);

Tools/TestNetRewarder/EventsFormatter.cs

@@ -15,6 +15,16 @@ namespace TestNetRewarder
         private readonly List<string> errors = new List<string>();
         private readonly EmojiMaps emojiMaps = new EmojiMaps();
 
+        public ChainEventMessage[] GetInitializationEvents(Configuration config)
+        {
+            return [
+                FormatBlock(0, "Bot initializing...",
+                    $"History-check start (UTC) = {Time.FormatTimestamp(config.HistoryStartUtc)}",
+                    $"Update interval = {Time.FormatDuration(config.Interval)}"
+                )
+            ];
+        }
+
         public ChainEventMessage[] GetEvents()
         {
             var result = events.ToArray();

Tools/TestNetRewarder/Processor.cs

@@ -11,11 +11,13 @@ namespace TestNetRewarder
         private readonly RewardChecker rewardChecker;
         private readonly EventsFormatter eventsFormatter;
         private readonly ChainState chainState;
+        private readonly Configuration config;
         private readonly BotClient client;
         private readonly ILog log;
 
         public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log)
         {
+            this.config = config;
             this.client = client;
             this.log = log;
 
@@ -31,6 +33,16 @@ namespace TestNetRewarder
             chainState = new ChainState(log, contracts, handler, config.HistoryStartUtc);
         }
 
+        public async Task Initialize()
+        {
+            var events = eventsFormatter.GetInitializationEvents(config);
+            var request = builder.Build(events, Array.Empty<string>());
+            if (request.HasAny())
+            {
+                await client.SendRewards(request);
+            }
+        }
+
         public async Task<TimeSegmentResponse> OnNewSegment(TimeRange timeRange)
         {
             try

Tools/TestNetRewarder/Program.cs

@@ -45,6 +45,7 @@ namespace TestNetRewarder
 
             Log.Log("Starting TestNet Rewarder...");
             var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor);
+            await processor.Initialize();
          
             while (!CancellationToken.IsCancellationRequested)
             {

Tools/TestNetRewarder/RewardCheck.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.ChainMonitor;
+using BlockchainUtils;
+using CodexContractsPlugin.ChainMonitor;
 using DiscordRewards;
 using GethPlugin;
 using NethereumWorkflow;

cs-codex-dist-testing.sln

@@ -80,6 +80,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexReleaseTests", "Tests\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExperimentalTests", "Tests\ExperimentalTests\ExperimentalTests.csproj", "{BA7369CD-7C2F-4075-8E35-98BCC19EE203}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BlockchainUtils", "Framework\BlockchainUtils\BlockchainUtils.csproj", "{4648B5AA-A0A7-44BA-89BC-2FD57370943C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SpeedCheckTests", "SpeedCheckTests\SpeedCheckTests.csproj", "{5D134D9A-DC61-4472-8F47-92250C8DB5CC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -214,6 +218,14 @@ Global
 		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BA7369CD-7C2F-4075-8E35-98BCC19EE203}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4648B5AA-A0A7-44BA-89BC-2FD57370943C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5D134D9A-DC61-4472-8F47-92250C8DB5CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5D134D9A-DC61-4472-8F47-92250C8DB5CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5D134D9A-DC61-4472-8F47-92250C8DB5CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5D134D9A-DC61-4472-8F47-92250C8DB5CC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -251,6 +263,8 @@ Global
 		{6230347F-5045-4E25-8E7A-13D7221B7444} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3}
 		{639A0603-4E80-465B-BB59-AB02F1DEEF5A} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 		{BA7369CD-7C2F-4075-8E35-98BCC19EE203} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
+		{4648B5AA-A0A7-44BA-89BC-2FD57370943C} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7}
+		{5D134D9A-DC61-4472-8F47-92250C8DB5CC} = {88C2A621-8A98-4D07-8625-7900FC8EF89E}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C}