Creates new overview after 3 new changes.

Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -75,11 +75,12 @@ namespace CodexContractsPlugin.ChainMonitor
                 throw new Exception(msg);
             }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "39-0C-32-A3-EA-90-4F-29-1C-67-12-F1-D5-BE-31-67-8D-90-43-1E-F2-02-63-5B-0C-49-F7-1E-E5-EC-F7-00";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -63,10 +63,10 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
@@ -82,7 +82,20 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync("", "")));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -261,4 +274,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -109,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -18,6 +18,7 @@ namespace CodexPlugin
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
@@ -138,6 +139,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +151,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -21,6 +21,14 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
+            };
+        }
+
         public LocalDatasetList Map(CodexOpenApi.DataList dataList)
         {
             return new LocalDatasetList
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -182,8 +199,20 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -243,4 +272,42 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -40,6 +40,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -344,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -359,15 +359,18 @@ components:
           description: "Indicates if content is protected by erasure-coding"
         filename:
           type: string
+          nullable: true
           description: "The original name of the uploaded content (optional)"
           example: codex.png
         mimetype:
           type: string
+          nullable: true
           description: "The original mimetype of the uploaded content (optional)"
           example: image/png
         uploadedAt:
           type: integer
           format: int64
+          nullable: true
           description: "The UTC upload timestamp in seconds"
           example: 1729244192
 
@@ -443,21 +446,6 @@ paths:
       summary: "Lists manifest CIDs stored locally in node."
       tags: [ Data ]
       operationId: listData
-      parameters:
-        - name: content-type
-          in: header
-          required: false
-          description: The content type of the file. Must be valid.
-          schema:
-            type: string
-            example: "image/png"
-        - name: content-disposition
-          in: header
-          required: false
-          description: The content disposition used to send the filename.
-          schema:
-            type: string
-            example: "attachment; filename=\"codex.png\""
       responses:
         "200":
           description: Retrieved list of content CIDs
@@ -478,6 +466,21 @@ paths:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:

Tools/AutoClient/App.cs

@@ -1,4 +1,5 @@
-using Logging;
+using AutoClient.Modes.FolderStore;
+using Logging;
 
 namespace AutoClient
 {
@@ -19,6 +20,15 @@ namespace AutoClient
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
+
+            if (!string.IsNullOrEmpty(config.FolderToStore))
+            {
+                FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
+            }
+            else
+            {
+                FolderWorkDispatcher = null!;
+            }
         }
 
         public Configuration Config { get; }
@@ -27,6 +37,7 @@ namespace AutoClient
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public CidRepo CidRepo { get; }
         public Performance Performance { get; }
+        public FolderWorkDispatcher FolderWorkDispatcher { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -0,0 +1,165 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class AutomaticPurchaser
+    {
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly CodexNode codex;
+        private Task workerTask = Task.CompletedTask;
+        private App app => instance.App;
+
+        public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
+        {
+            this.log = log;
+            this.instance = instance;
+            this.codex = codex;
+        }
+
+        public void Start()
+        {
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
+        }
+
+        private async Task Worker()
+        {
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(instance.NodeId);
+            if (cid == null) return;
+
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+            await codex.DownloadCid(filename, cid, size);
+
+            DeleteFile(filename);
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            try
+            {
+                var cid = await codex.UploadFile(file);
+                var response = await codex.RequestStorage(cid);
+                return response.PurchaseId;
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (!app.Cts.Token.IsCancellationRequested)
+                {
+                    var purchase = await codex.GetStoragePurchase(pid);
+                    if (purchase == null)
+                    {
+                        await FixedShortDelay();
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                        continue;
+                    }
+                    if (purchase.IsCancelled)
+                    {
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (purchase.IsError)
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (purchase.IsFinished)
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (purchase.IsStarted)
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, app.Cts.Token);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -17,6 +17,7 @@
             lock (_lock)
             {
                 entries.Add(new CidEntry(nodeId, cid, knownSize));
+                if (entries.Count > 1000) entries.Clear();
             }
         }
 

Tools/AutoClient/CodexInstance.cs

@@ -0,0 +1,161 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Nethereum.Model;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public interface ICodexInstance
+    {
+        string NodeId { get; }
+        App App { get; }
+        CodexApi Codex { get; }
+        HttpClient Client { get; }
+        Address Address { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            App = app;
+            Codex = codex;
+            Client = client;
+            Address = address;
+            NodeId = Guid.NewGuid().ToString();
+        }
+
+        public string NodeId { get; }
+        public App App { get; }
+        public CodexApi Codex { get; }
+        public HttpClient Client { get; }
+        public Address Address { get; }
+    }
+
+    public class CodexNode
+    {
+        private readonly App app;
+        private readonly ICodexInstance codex;
+
+        public CodexNode(App app, ICodexInstance instance)
+        {
+            this.app = app;
+            codex = instance;
+        }
+
+        public async Task DownloadCid(string filename, string cid, long? size)
+        {
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                using var fileStream = File.OpenWrite(filename);
+                var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
+                fileResponse.Stream.CopyTo(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.DownloadSuccessful(size, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
+            }
+        }
+
+        public async Task<ContentId> UploadFile(string filename)
+        {
+            using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream, filename);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
+
+        public async Task<RequestStorageResult> RequestStorage(ContentId cid)
+        {
+            app.Log.Debug("Requesting storage for " + cid.Id);
+            var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
+
+            app.Log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
+
+            return new RequestStorageResult(result, new ContentId(encoded));
+        }
+
+        public class RequestStorageResult
+        {
+            public RequestStorageResult(string purchaseId, ContentId encodedCid)
+            {
+                PurchaseId = purchaseId;
+                EncodedCid = encodedCid;
+            }
+
+            public string PurchaseId { get; }
+            public ContentId EncodedCid { get; }
+
+            public override string ToString()
+            {
+                return $"{PurchaseId} (cid: {EncodedCid})";
+            }
+        }
+
+        public async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
+        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
+        {
+            app.Log.Debug($"Uploading file...");
+            var response = await codex.Codex.UploadAsync(
+                content_type: "application/octet-stream",
+                content_disposition: $"attachment; filename=\"{filename}\"",
+                fileStream, app.Cts.Token);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> GetEncodedCid(string pid)
+        {
+            try
+            {
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
+            }
+            catch (Exception ex)
+            {
+                app.Log.Error(ex.ToString());
+                throw;
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -5,7 +5,8 @@ namespace AutoClient
     public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
-        public string CodexEndpoints { get; set; } = "http://localhost:8080";
+        public string CodexEndpoints { get; set; } =
+            "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -14,19 +15,22 @@ namespace AutoClient
         public int NumConcurrentPurchases { get; set; } = 10;
 
         [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
-        public int ContractDurationMinutes { get; set; } = 60 * 6;
+        public int ContractDurationMinutes { get; set; } =
+            60 * 24 * 6; // 6 days
+            //60 * 6; 6 hours
+        // Cluster nodes configured for max 7-day storage.
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 15;
+        public int ContractExpiryMinutes { get; set; } = 60;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
-        public int NumHosts { get; set; } = 10;
+        public int NumHosts { get; set; } = 5;
 
         [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
-        public int HostTolerance { get; set; } = 5;
+        public int HostTolerance { get; set; } = 1;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
-        public int Price { get; set; } = 10;
+        public int Price { get; set; } = 1000;
 
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
@@ -34,6 +38,9 @@ namespace AutoClient
         [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
         public int FileSizeMb { get; set; } = 0;
 
+        [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
+        public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
+
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -0,0 +1,58 @@
+using static AutoClient.Modes.FolderStore.FileWorker;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileStatus : JsonBacked<WorkerStatus>
+    {
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
+            : base(app, folder, filePath + ".json")
+        {
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public bool IsBusy()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                !p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyRunning()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyFailed()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            var mostRecent = GetMostRecent();
+            if (mostRecent == null) return false;
+
+            return mostRecent.Expiry.HasValue;
+        }
+
+        protected WorkerPurchase? GetMostRecent()
+        {
+            if (!State.Purchases.Any()) return null;
+            var maxCreated = State.Purchases.Max(p => p.Created);
+            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -0,0 +1,293 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileWorker : FileStatus
+    {
+        private readonly App app;
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly string sourceFilename;
+        private readonly Action onFileUploaded;
+        private readonly Action onNewPurchase;
+        private readonly CodexNode codex;
+
+        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
+            : base(app, folder, fileIndex.File + ".json", purchaseInfo)
+        {
+            this.app = app;
+            log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
+            this.instance = instance;
+            this.purchaseInfo = purchaseInfo;
+            sourceFilename = fileIndex.File;
+            if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
+            this.onFileUploaded = onFileUploaded;
+            this.onNewPurchase = onNewPurchase;
+            codex = new CodexNode(app, instance);
+        }
+
+        public int FailureCounter => State.FailureCounter;
+
+        protected override void OnNewState(WorkerStatus newState)
+        {
+            newState.LastUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update()
+        {
+            try
+            {
+                if (IsCurrentlyRunning() && UpdatedRecently()) return;
+
+                Log($"Updating for '{sourceFilename}'...");
+                await EnsureRecentPurchase();
+                SaveState();
+                app.Log.Log("");
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Exception during fileworker update: " + exc);
+                State.Error = exc.ToString();
+                SaveState();
+                throw;
+            }
+        }
+
+        private bool UpdatedRecently()
+        {
+            var now = DateTime.UtcNow;
+            return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
+        }
+
+        private async Task<string> EnsureCid()
+        {
+            Log($"Checking CID...");
+
+            if (!string.IsNullOrEmpty(State.EncodedCid) &&
+                await DoesCidExistInNetwork(State.EncodedCid))
+            {
+                Log("Encoded-CID successfully found in the network.");
+                // TODO: Using the encoded CID currently would result in double-encoding of the dataset.
+                // See: https://github.com/codex-storage/nim-codex/issues/1005
+                // Always use the basic CID for now, even though we have to repeat the encoding.
+                // When using encoded CID works: return State.EncodedCid;
+            }
+
+            if (!string.IsNullOrEmpty(State.Cid) &&
+                await DoesCidExistInNetwork(State.Cid))
+            {
+                Log("Basic-CID successfully found in the network.");
+                return State.Cid;
+            }
+
+            if (string.IsNullOrEmpty(State.Cid))
+            {
+                Log("File was not previously uploaded.");
+            }
+
+            Log($"Uploading...");
+            var cid = await codex.UploadFile(sourceFilename);
+            onFileUploaded();
+            Log("Got Basic-CID: " + cid);
+            State.Cid = cid.Id;
+            SaveState();
+            return State.Cid;
+        }
+
+        private async Task<bool> DoesCidExistInNetwork(string cid)
+        {
+            try
+            {
+                // This should not take longer than a few seconds. If it does, cancel it.
+                var cts = new CancellationTokenSource();
+                var cancelTask = Task.Run(() =>
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(15));
+                    cts.Cancel();
+                });
+
+                var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
+                if (manifest == null) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+        private async Task EnsureRecentPurchase()
+        {
+            Log($"Checking recent purchase...");
+            var recent = GetMostRecent();
+            if (recent == null)
+            {
+                Log($"No recent purchase.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            await UpdatePurchase(recent);
+
+            if (recent.Expiry.HasValue)
+            {
+                Log($"Purchase has failed or expired.");
+                await MakeNewPurchase();
+                State.FailureCounter++;
+                return;
+            }
+
+            if (recent.Finish.HasValue)
+            {
+                Log($"Purchase has finished.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
+            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
+            {
+                Log($"Purchase is going to expire soon.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            if (!recent.Submitted.HasValue)
+            {
+                Log($"Purchase is waiting to be submitted.");
+                return;
+            }
+
+            if (recent.Submitted.HasValue && !recent.Started.HasValue)
+            {
+                Log($"Purchase is submitted and waiting to start.");
+                return;
+            }
+
+            Log($"Purchase is running.");
+        }
+
+        private async Task UpdatePurchase(WorkerPurchase recent)
+        {
+            if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
+            var now = DateTime.UtcNow;
+
+            var purchase = await codex.GetStoragePurchase(recent.Pid);
+            if (purchase == null)
+            {
+                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
+                recent.Expiry = now;
+                return;
+            }
+
+            if (purchase.IsSubmitted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+            }
+            if (purchase.IsStarted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+            }
+            if (purchase.IsCancelled)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsError)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsFinished)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+                if (!recent.Finish.HasValue) recent.Finish = now;
+            }
+            State.LastUpdate = now;
+            SaveState();
+        }
+
+        private async Task MakeNewPurchase()
+        {
+            var cid = await EnsureCid();
+            if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
+
+            Log($"Creating new purchase...");
+            var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
+            var purchaseId = response.PurchaseId;
+            var encodedCid = response.EncodedCid;
+            if (string.IsNullOrEmpty(purchaseId) ||
+                purchaseId == "Unable to encode manifest" ||
+                purchaseId == "Purchasing not available" ||
+                purchaseId == "Expiry required" ||
+                purchaseId == "Expiry needs to be in future" ||
+                purchaseId == "Expiry has to be before the request's end (now + duration)")
+            {
+                throw new InvalidOperationException(purchaseId);
+            }
+
+            var newPurchase = new WorkerPurchase
+            {
+                Created = DateTime.UtcNow,
+                Pid = purchaseId
+            };
+            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
+            State.EncodedCid = encodedCid.Id;
+            SaveState();
+            onNewPurchase();
+
+            Log($"New purchase created. PID: '{purchaseId}'.");
+            Log("Got Encoded-CID: " + encodedCid);
+            Log("Waiting for submit...");
+            Thread.Sleep(500);
+
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (DateTime.UtcNow < timeout)
+            {
+                Thread.Sleep(5000);
+                await UpdatePurchase(newPurchase);
+                if (newPurchase.Submitted.HasValue)
+                {
+                    Log("New purchase successfully submitted.");
+                    return;
+                }
+            }
+            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
+        private string GetFileTag(FileIndex filename)
+        {
+            return $"({filename.Index.ToString("00000")}) ";
+        }
+
+        [Serializable]
+        public class WorkerStatus
+        {
+            public DateTime LastUpdate { get; set; }
+            public string Cid { get; set; } = string.Empty;
+            public string EncodedCid { get; set; } = string.Empty;
+            public int FailureCounter { get; set; } = 0;
+            public string Error { get; set; } = string.Empty;
+            public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
+        }
+
+        [Serializable]
+        public class WorkerPurchase
+        {
+            public string Pid { get; set; } = string.Empty;
+            public DateTime Created { get; set; }
+            public DateTime? Submitted { get; set; }
+            public DateTime? Started { get; set; }
+            public DateTime? Expiry { get; set; }
+            public DateTime? Finish { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -0,0 +1,70 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkDispatcher
+    {
+        private readonly string[] files = Array.Empty<string>();
+        private readonly ILog log;
+        private int index = 0;
+        private int busyCount = 0;
+
+        public FolderWorkDispatcher(ILog log, string folder)
+        {
+            var fs = Directory.GetFiles(folder);
+            var result = new List<string>();
+            foreach (var f in fs)
+            {
+                if (!f.ToLowerInvariant().Contains(".json"))
+                {
+                    var info = new FileInfo(f);
+                    if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
+                    {
+                        result.Add(f);
+                    }
+                }
+            }
+            files = result.ToArray();
+            this.log = log;
+        }
+
+        public FileIndex GetFileToCheck()
+        {
+            if (busyCount > 0)
+            {
+                log.Log("");
+                log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
+                log.Log("");
+                ResetIndex();
+                Thread.Sleep(TimeSpan.FromMinutes(1));
+            }
+
+            var file = new FileIndex(files[index], index);
+            index = (index + 1) % files.Length;
+            return file;
+        }
+
+        public void ResetIndex()
+        {
+            index = 0;
+            busyCount = 0;
+        }
+
+        public void WorkerIsBusy()
+        {
+            busyCount++;
+        }
+    }
+
+    public class FileIndex
+    {
+        public FileIndex(string file, int index)
+        {
+            File = file;
+            Index = index;
+        }
+
+        public string File { get; }
+        public int Index { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -0,0 +1,136 @@
+using CodexOpenApi;
+using System.IO.Compression;
+using static AutoClient.Modes.FolderStore.FolderWorkOverview;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
+    {
+        private const string OverviewFilename = "codex_folder_saver_overview.json";
+        private readonly App app;
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
+            : base(app, folder, Path.Combine(folder, OverviewFilename))
+        {
+            this.app = app;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        protected override void OnNewState(WorkMonitorStatus newState)
+        {
+            newState.LastOverviewUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update(ICodexInstance instance)
+        {
+            var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
+
+            var total = 0;
+            var successful = 0;
+            var failed = 0;
+            foreach (var file in jsonFiles)
+            {
+                try
+                {
+                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
+                    total++;
+                    if (worker.IsCurrentlyRunning()) successful++;
+                    if (worker.IsCurrentlyFailed()) failed++;
+                }
+                catch (Exception exc)
+                {
+                    app.Log.Error("Exception in workoverview update: " + exc);
+                }
+            }
+
+            State.TotalFiles = total;
+            State.SuccessfulStored = successful;
+            State.StoreFailed = failed;
+            SaveState();
+
+            if (State.UncommitedChanges > 3)
+            {
+                State.UncommitedChanges = 0;
+                SaveState();
+
+                await CreateNewOverviewZip(jsonFiles, FilePath, instance);
+            }
+        }
+
+        public void MarkUncommitedChange()
+        {
+            State.UncommitedChanges++;
+            SaveState();
+        }
+
+        private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
+        {
+            Log("");
+            Log("");
+            Log("Creating new overview zipfile...");
+            var zipFilename = CreateZipFile(jsonFiles, filePath);
+
+            Log("Uploading to Codex...");
+            try
+            {
+                var codex = new CodexNode(app, instance);
+                var cid = await codex.UploadFile(zipFilename);
+                Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
+                Log("Requesting storage for it...");
+                var result = await codex.RequestStorage(cid);
+                Log("Storage requested. Purchase ID: " + result);
+
+                var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
+                File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
+                Log($">>> [{outFile}] has been updated. <<<");
+            }
+            catch (Exception exc)
+            {
+                Log("Failed to upload new overview zipfile: " + exc);
+            }
+            Log("");
+            Log("");
+        }
+
+        private string CreateZipFile(List<string> jsonFiles, string filePath)
+        {
+            var zipFilename = Guid.NewGuid().ToString() + ".zip";
+
+            using (var memoryStream = new MemoryStream())
+            {
+                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
+                {
+                    archive.CreateEntryFromFile(filePath, "overview.json");
+                    foreach (var file in jsonFiles)
+                    {
+                        archive.CreateEntryFromFile(file, Path.GetFileName(file));
+                    }
+                }
+
+                using (var fileStream = new FileStream(zipFilename, FileMode.Create))
+                {
+                    memoryStream.Seek(0, SeekOrigin.Begin);
+                    memoryStream.CopyTo(fileStream);
+                }
+            }
+            return zipFilename;
+        }
+
+        private void Log(string msg)
+        {
+            app.Log.Log(msg);
+        }
+
+        [Serializable]
+        public class WorkMonitorStatus
+        {
+            public int TotalFiles { get; set; }
+            public int SuccessfulStored { get; set; }
+            public int StoreFailed { get; set; }
+
+            public DateTime LastOverviewUpdate { get; set; }
+            public int UncommitedChanges { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/JsonBacked.cs

@@ -0,0 +1,58 @@
+using Newtonsoft.Json;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public abstract class JsonBacked<T> where T : new()
+    {
+        private readonly App app;
+
+        protected JsonBacked(App app, string folder, string filePath)
+        {
+            this.app = app;
+            Folder = folder;
+            FilePath = filePath;
+            LoadState();
+        }
+
+        private void LoadState()
+        {
+            try
+            {
+                if (!File.Exists(FilePath))
+                {
+                    State = new T();
+                    OnNewState(State);
+                    SaveState();
+                }
+                var text = File.ReadAllText(FilePath);
+                State = JsonConvert.DeserializeObject<T>(text)!;
+                if (State == null) throw new Exception("Didn't deserialize " + FilePath);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to load state: " + exc);
+            }
+        }
+
+        protected string Folder { get; }
+        protected string FilePath { get; }
+        protected T State { get; private set; } = default!;
+
+        protected virtual void OnNewState(T newState)
+        {
+        }
+
+        protected void SaveState()
+        {
+            try
+            {
+                var json = JsonConvert.SerializeObject(State);
+                File.WriteAllText(FilePath, json);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to save state: " + exc);
+            }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/PurchaseInfo.cs

@@ -0,0 +1,18 @@
+namespace AutoClient.Modes.FolderStore
+{
+    public class PurchaseInfo
+    {
+        public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
+        {
+            PurchaseDurationTotal = purchaseDurationTotal;
+            PurchaseDurationSafe = purchaseDurationSafe;
+
+            if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
+            if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
+            if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
+        }
+
+        public TimeSpan PurchaseDurationTotal { get; }
+        public TimeSpan PurchaseDurationSafe { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -0,0 +1,86 @@
+using AutoClient.Modes.FolderStore;
+
+namespace AutoClient.Modes
+{
+    public class FolderStoreMode : IMode
+    {
+        private readonly App app;
+        private readonly string folder;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly CancellationTokenSource cts = new CancellationTokenSource();
+        private Task checkTask = Task.CompletedTask;
+
+        public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
+        {
+            this.app = app;
+            this.folder = folder;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public void Start(ICodexInstance instance, int index)
+        {
+            checkTask = Task.Run(async () =>
+            {
+                try
+                {
+                    await RunChecker(instance);
+                }
+                catch (Exception ex)
+                {
+                    app.Log.Error("Exception in FolderStoreMode worker: " + ex);
+                    Environment.Exit(1);
+                }
+            });
+        }
+
+        private async Task RunChecker(ICodexInstance instance)
+        {
+            var i = 0;
+            while (!cts.IsCancellationRequested)
+            {
+                Thread.Sleep(2000);
+
+                var worker = await ProcessWorkItem(instance);
+                if (worker.FailureCounter > 5)
+                {
+                    throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
+                }
+                i++;
+
+                if (i > 5)
+                {
+                    i = 0;
+                    var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+                    await overview.Update(instance);
+                }
+            }
+        }
+
+        private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
+        {
+            var file = app.FolderWorkDispatcher.GetFileToCheck();
+            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
+            await worker.Update();
+            if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
+            return worker;
+        }
+
+        private void OnFileUploaded()
+        {
+        }
+
+        private void OnNewPurchase()
+        {
+            app.FolderWorkDispatcher.ResetIndex();
+
+            var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+            overview.MarkUncommitedChange();
+        }
+
+        public void Stop()
+        {
+            cts.Cancel();
+            checkTask.Wait();
+        }
+    }
+}

Tools/AutoClient/Modes/Mode.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AutoClient.Modes
+{
+    public interface IMode
+    {
+        void Start(ICodexInstance instance, int index);
+        void Stop();
+    }
+}

Tools/AutoClient/Modes/PurchasingMode.cs

@@ -1,32 +1,23 @@
-using CodexOpenApi;
-using Logging;
-using Utils;
+using Logging;
 
-namespace AutoClient
+namespace AutoClient.Modes
 {
-    public class CodexUser
+    public class PurchasingMode : IMode
     {
+        private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
         private readonly App app;
-        private readonly CodexApi codex;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly List<Purchaser> purchasers = new List<Purchaser>();
         private Task starterTask = Task.CompletedTask;
-        private readonly string nodeId = Guid.NewGuid().ToString();
 
-        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        public PurchasingMode(App app)
         {
             this.app = app;
-            this.codex = codex;
-            this.client = client;
-            this.address = address;
         }
 
-        public void Start(int index)
+        public void Start(ICodexInstance instance, int index)
         {
             for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
             {
-                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+                purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
             }
 
             var delayPerPurchaser =

Tools/AutoClient/Performance.cs

@@ -16,11 +16,13 @@ namespace AutoClient
             Log($"Download failed: {ex}");
         }
 
-        public void DownloadSuccessful(long size, TimeSpan time)
+        public void DownloadSuccessful(long? size, TimeSpan time)
         {
+            if (!size.HasValue) return;
+
             long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
             if (milliseconds < 1) milliseconds = 1;
-            long bytesPerSecond = 1000 * (size / milliseconds);
+            long bytesPerSecond = 1000 * (size.Value / milliseconds);
             Log($"Download successful: {bytesPerSecond} bytes per second");
         }
 

Tools/AutoClient/Program.cs

@@ -1,11 +1,14 @@
 using ArgsUniform;
 using AutoClient;
+using AutoClient.Modes;
+using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;
 
 public class Program
 {
     private readonly App app;
+    private readonly List<IMode> modes = new List<IMode>();
 
     public Program(Configuration config)
     {
@@ -31,36 +34,60 @@ public class Program
 
     public async Task Run()
     {
-        var codexUsers = await CreateUsers();
+        var codexInstances = await CreateCodexInstances();
 
         var i = 0;
-        foreach (var user in codexUsers)
+        foreach (var cdx in codexInstances)
         {
-            user.Start(i);
+            var mode = CreateMode();
+            modes.Add(mode);
+
+            mode.Start(cdx, i);
             i++;
         }
 
         app.Cts.Token.WaitHandle.WaitOne();
 
-        foreach (var user in codexUsers) user.Stop();
+        foreach (var mode in modes) mode.Stop();
+        modes.Clear();
 
         app.Log.Log("Done");
     }
 
-    private async Task<CodexUser[]> CreateUsers()
+    private IMode CreateMode()
+    {
+        if (!string.IsNullOrEmpty(app.Config.FolderToStore))
+        {
+            return CreateFolderStoreMode();
+        }
+
+        return new PurchasingMode(app);
+    }
+
+    private IMode CreateFolderStoreMode()
+    {
+        if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
+
+        return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
+            purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
+            purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
+        ));
+    }
+
+    private async Task<CodexInstance[]> CreateCodexInstances()
     {
         var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
-        var result = new List<CodexUser>();
+        var result = new List<CodexInstance>();
 
         foreach (var e in endpointStrs)
         {
-            result.Add(await CreateUser(e));
+            result.Add(await CreateCodexInstance(e));
         }
 
         return result.ToArray();
     }
 
-    private async Task<CodexUser> CreateUser(string endpoint)
+    private async Task<CodexInstance> CreateCodexInstance(string endpoint)
     {
         var splitIndex = endpoint.LastIndexOf(':');
         var host = endpoint.Substring(0, splitIndex);
@@ -72,6 +99,7 @@ public class Program
         );
 
         var client = new HttpClient();
+        client.Timeout = TimeSpan.FromMinutes(60.0);
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
@@ -79,7 +107,7 @@ public class Program
         await CheckCodex(codex);
         app.Log.Log("OK");
 
-        return new CodexUser(
+        return new CodexInstance(
             app,
             codex,
             client,
@@ -105,4 +133,4 @@ public class Program
     {
         Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
     }
-}
+}

Tools/AutoClient/Purchaser.cs

@@ -1,258 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public class Purchaser
-    {
-        private readonly App app;
-        private readonly string nodeId;
-        private readonly ILog log;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly CodexApi codex;
-        private Task workerTask = Task.CompletedTask;
-
-        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
-        {
-            this.app = app;
-            this.nodeId = nodeId;
-            this.log = log;
-            this.client = client;
-            this.address = address;
-            this.codex = codex;
-        }
-
-        public void Start()
-        {
-            workerTask = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            workerTask.Wait();
-        }
-
-        private async Task Worker()
-        {
-            log.Log("Worker started.");
-            while (!app.Cts.Token.IsCancellationRequested)
-            {
-                try
-                {
-                    var pid = await StartNewPurchase();
-                    await WaitTillFinished(pid);
-                    await DownloadForeignCid();
-                }
-                catch (Exception ex)
-                {
-                    log.Error("Worker failed with: " + ex);
-                    await Task.Delay(TimeSpan.FromHours(6));
-                }
-            }
-        }
-
-        private async Task DownloadForeignCid()
-        {
-            var cid = app.CidRepo.GetForeignCid(nodeId);
-            if (cid == null) return;
-            var size = app.CidRepo.GetSizeForCid(cid);
-            if (size == null) return;
-
-            try
-            {
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
-                {
-                    using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
-                    fileResponse.Stream.CopyTo(fileStream);
-                }
-                var time = sw.Elapsed;
-                File.Delete(filename);
-                app.Performance.DownloadSuccessful(size.Value, time);
-            }
-            catch (Exception ex)
-            {
-                app.Performance.DownloadFailed(ex);
-            }
-        }
-
-        private async Task<string> StartNewPurchase()
-        {
-            var file = await CreateFile();
-            try
-            {
-                var cid = await UploadFile(file);
-                return await RequestStorage(cid);
-            }
-            finally
-            {
-                DeleteFile(file);
-            }
-        }
-
-        private async Task<string> CreateFile()
-        {
-            return await app.Generator.Generate();
-        }
-
-        private void DeleteFile(string file)
-        {
-            try
-            {
-                File.Delete(file);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error($"Failed to delete file '{file}': {exc}");
-            }
-        }
-
-        private async Task<ContentId> UploadFile(string filename)
-        {
-            using var fileStream = File.OpenRead(filename);
-            try
-            {
-                var info = new FileInfo(filename);
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream);
-                var time = sw.Elapsed;
-                app.Performance.UploadSuccessful(info.Length, time);
-                app.CidRepo.Add(nodeId, cid.Id, info.Length);
-                return cid;
-            }
-            catch (Exception exc)
-            {
-                app.Performance.UploadFailed(exc);
-                throw;
-            }
-        }
-
-        private async Task<ContentId> UploadStream(FileStream fileStream)
-        {
-            log.Debug($"Uploading file...");
-            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
-
-            log.Debug($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        private async Task<string> RequestStorage(ContentId cid)
-        {
-            log.Debug("Requesting storage for " + cid.Id);
-            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
-            {
-                Collateral = app.Config.RequiredCollateral.ToString(),
-                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = app.Config.NumHosts,
-                Reward = app.Config.Price.ToString(),
-                ProofProbability = "15",
-                Tolerance = app.Config.HostTolerance
-            }, app.Cts.Token);
-
-            log.Debug("Purchase ID: " + result);
-
-            var encoded = await GetEncodedCid(result);
-            app.CidRepo.AddEncoded(cid.Id, encoded);
-
-            return result;
-        }
-
-        private async Task<string> GetEncodedCid(string pid)
-        {
-            try
-            {
-                var sp = (await GetStoragePurchase(pid))!;
-                return sp.Request.Content.Cid;
-            }
-            catch (Exception ex)
-            {
-                log.Error(ex.ToString());
-                throw;
-            }
-        }
-
-        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
-        {
-            // openapi still don't match code.
-            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-            if (string.IsNullOrEmpty(str)) return null;
-            return JsonConvert.DeserializeObject<StoragePurchase>(str);
-        }
-
-        private async Task WaitTillFinished(string pid)
-        {
-            try
-            {
-                var emptyResponseTolerance = 10;
-                while (!app.Cts.Token.IsCancellationRequested)
-                {
-                    var purchase = await GetStoragePurchase(pid);
-                    if (purchase == null)
-                    {
-                        await FixedShortDelay();
-                        emptyResponseTolerance--;
-                        if (emptyResponseTolerance == 0)
-                        {
-                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
-                            await ExpiryTimeDelay();
-                            return;
-                        }
-                        continue;
-                    }
-                    var status = purchase.State.ToLowerInvariant();
-                    if (status.Contains("cancel"))
-                    {
-                        app.Performance.StorageContractCancelled();
-                        return;
-                    }
-                    if (status.Contains("error"))
-                    {
-                        app.Performance.StorageContractErrored(purchase.Error);
-                        return;
-                    }
-                    if (status.Contains("finished"))
-                    {
-                        app.Performance.StorageContractFinished();
-                        return;
-                    }
-                    if (status.Contains("started"))
-                    {
-                        app.Performance.StorageContractStarted();
-                        await FixedDurationDelay();
-                    }
-
-                    await FixedShortDelay();
-                }
-            }
-            catch (Exception ex)
-            {
-                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
-                await ExpiryTimeDelay();
-            }
-        }
-
-        private async Task FixedDurationDelay()
-        {
-            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task ExpiryTimeDelay()
-        {
-            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task FixedShortDelay()
-        {
-            await Task.Delay(15 * 1000, app.Cts.Token);
-        }
-    }
-}

Tools/BiblioTech/CodexCidChecker.cs

@@ -1,5 +1,6 @@
 using CodexOpenApi;
 using IdentityModel.Client;
+using Logging;
 using Utils;
 
 namespace BiblioTech
@@ -8,11 +9,13 @@ namespace BiblioTech
     {
         private static readonly string nl = Environment.NewLine;
         private readonly Configuration config;
+        private readonly ILog log;
         private CodexApi? currentCodexNode;
 
-        public CodexCidChecker(Configuration config)
+        public CodexCidChecker(Configuration config, ILog log)
         {
             this.config = config;
+            this.log = log;
         }
 
         public async Task<CheckResponse> PerformCheck(string cid)
@@ -63,7 +66,7 @@ namespace BiblioTech
                 success: true,
                 title: $"Success: '{content.Cid}'",
                 error: "",
-                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"size: {content.Manifest.DatasetSize} bytes",
                 $"blockSize: {content.Manifest.BlockSize} bytes",
                 $"protected: {content.Manifest.Protected}"
             );
@@ -150,6 +153,7 @@ namespace BiblioTech
             }
             catch (Exception e)
             {
+                log.Error(e.ToString());
                 return false;
             }
         }

Tools/BiblioTech/Program.cs

@@ -80,7 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checker = new CodexCidChecker(Config);
+            var checker = new CodexCidChecker(Config, Log);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();