wip tracking download fails

# Conflicts:
#	ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

Framework/KubernetesWorkflow/LogHandler.cs

@@ -1,4 +1,5 @@
 using Logging;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -40,6 +41,11 @@ namespace KubernetesWorkflow
 
         protected override void ProcessLine(string line)
         {
+            foreach (var replacement in BaseLog.replacements)
+            {
+                line = replacement.Apply(line);
+            }
+
             LogFile.WriteRaw(line);
         }
     }

Framework/Logging/BaseLog.cs

@@ -16,7 +16,7 @@ namespace Logging
         public static bool EnableDebugLogging { get; set; } = false;
 
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
+        public static List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private LogFile? logFile;
 
         public BaseLog()

Framework/Utils/BlockInterval.cs

@@ -15,13 +15,12 @@
                 To = from;
             }
             TimeRange = timeRange;
-            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks { get; }
+        public ulong NumberOfBlocks => To - From;
 
         public override string ToString()
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -75,12 +75,11 @@ namespace CodexContractsPlugin.ChainMonitor
                 throw new Exception(msg);
             }
 
-            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
+            log.Log($"ChainState updating: {events.BlockInterval}");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
-            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
+        private const string OpenApiYamlHash = "39-0C-32-A3-EA-90-4F-29-1C-67-12-F1-D5-BE-31-67-8D-90-43-1E-F2-02-63-5B-0C-49-F7-1E-E5-EC-F7-00";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -63,10 +63,10 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
+        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
+                api => api.UploadAsync(fileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
@@ -82,20 +82,7 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            // API for listData mismatches.
-            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
-
-            return mapper.Map(CrashCheck(() =>
-            {
-                var endpoint = GetEndpoint();
-                return Time.Retry(() =>
-                {
-                    var str = endpoint.HttpGetString("data");
-                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
-                }, nameof(LocalFiles));
-            }));
-
+            return mapper.Map(OnCodex(api => api.ListDataAsync("", "")));
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -274,18 +261,4 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
-
-    public class UploadInput
-    {
-        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
-        {
-            ContentType = contentType;
-            ContentDisposition = contentDisposition;
-            FileStream = fileStream;
-        }
-
-        public string ContentType { get; }
-        public string ContentDisposition { get; }
-        public FileStream FileStream { get; }
-    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,20 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
+        private const string DefaultDockerImage =
+        //"codexstorage/nim-codex:0.1.7-dist-tests"; // => 20/20: 17 seconds 10/10: 3 seconds
+        //"codexstorage/nim-codex:sha-2a25460-dist-tests"; // PR => 20/20: 17 seconds
+        //"thatbenbierens/nim-codex:blockexcpr1"; // PR with revert of "Fixes issue where only wants of type block are stored in peerContext" => 20/20: 17 seconds
+        //"thatbenbierens/nim-codex:blockexprecreate"; // v0.1.7 with patch => 20/20: 19 seconds
+        //"thatbenbierens/nim-codex:blockexprecreate016"; // v0.1.6 with patch => 20/20: 19 seconds 10/10: 2 seconds
+        //"thatbenbierens/nim-codex:blockexchprtinker7";
+        //"thatbenbierens/nim-codex:blkexc9"; // wow-fast
+
+        "thatbenbierens/nim-codex:asyncprofile5break"; // asynced trees.
+
+        //blocks are stored, blocks are resolved
+        //store-stream does not continue. node too busy???
+
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -109,7 +122,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
+                AddEnvVar("PRIV_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -18,7 +18,6 @@ namespace CodexPlugin
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
-        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
@@ -139,11 +138,6 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
-        {
-            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
-        }
-
-        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -151,11 +145,10 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var input = new UploadInput(contentType, contentDisposition, fileStream);
-            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
+            var logMessage = $"Uploading file {file.Describe()}...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(input, onFailure);
+                return CodexAccess.UploadFile(fileStream, onFailure);
             });
 
             var response = measurement.Value;

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -21,14 +21,6 @@ namespace CodexPlugin
             };
         }
 
-        public LocalDatasetList Map(LocalDatasetListJson json)
-        {
-            return new LocalDatasetList
-            {
-                Content = json.Content.Select(Map).ToArray()
-            };
-        }
-
         public LocalDatasetList Map(CodexOpenApi.DataList dataList)
         {
             return new LocalDatasetList
@@ -46,15 +38,6 @@ namespace CodexPlugin
             };
         }
 
-        public LocalDataset Map(LocalDatasetListJsonItem item)
-        {
-            return new LocalDataset
-            {
-                Cid = new ContentId(item.Cid),
-                Manifest = MapManifest(item.Manifest)
-            };
-        }
-
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -199,20 +182,8 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
-                RootHash = manifest.TreeCid,
-                Protected = manifest.Protected
-            };
-        }
-
-        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
-        {
-            return new Manifest
-            {
-                // needs update
-                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
-                RootHash = manifest.TreeCid,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
+                RootHash = manifest.RootHash,
                 Protected = manifest.Protected
             };
         }
@@ -272,42 +243,4 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
-
-
-//"content": [
-//        {
-//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
-//            "manifest": {
-//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
-//                "datasetSize": 5242880,
-//                "blockSize": 65536,
-//                "filename": null,
-//                "mimetype": "application/octet-stream",
-//                "uploadedAt": 1731426230,
-//                "protected": false
-//            }
-//        }
-//    ]
-
-    public class LocalDatasetListJson
-    {
-        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
-    }
-
-    public class LocalDatasetListJsonItem
-    {
-        public string Cid { get; set; } = string.Empty;
-        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
-    }
-
-    public class LocalDatasetListJsonItemManifest
-    {
-        public string TreeCid { get; set; } = string.Empty;
-        public int DatasetSize { get; set; }
-        public int BlockSize { get; set; }
-        public string? Filename { get; set; } = string.Empty;
-        public string? MimeType { get; set; } = string.Empty;
-        public int? UploadedAt { get; set; } 
-        public bool Protected { get; set; }
-    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -40,12 +40,6 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
-
-        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
-        public bool IsError => State.ToLowerInvariant().Contains("error");
-        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
-        public bool IsStarted => State.ToLowerInvariant().Contains("started");
-        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -344,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        treeCid:
+        rootHash:
           $ref: "#/components/schemas/Cid"
-          description: "Unique data identifier"
-        datasetSize:
+          description: "Root hash of the content"
+        originalBytes:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -359,18 +359,15 @@ components:
           description: "Indicates if content is protected by erasure-coding"
         filename:
           type: string
-          nullable: true
           description: "The original name of the uploaded content (optional)"
           example: codex.png
         mimetype:
           type: string
-          nullable: true
           description: "The original mimetype of the uploaded content (optional)"
           example: image/png
         uploadedAt:
           type: integer
           format: int64
-          nullable: true
           description: "The UTC upload timestamp in seconds"
           example: 1729244192
 
@@ -446,6 +443,21 @@ paths:
       summary: "Lists manifest CIDs stored locally in node."
       tags: [ Data ]
       operationId: listData
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       responses:
         "200":
           description: Retrieved list of content CIDs
@@ -466,21 +478,6 @@ paths:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
-      parameters:
-        - name: content-type
-          in: header
-          required: false
-          description: The content type of the file. Must be valid.
-          schema:
-            type: string
-            example: "image/png"
-        - name: content-disposition
-          in: header
-          required: false
-          description: The content disposition used to send the filename.
-          schema:
-            type: string
-            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:

Tools/AutoClient/App.cs

@@ -1,5 +1,4 @@
-using AutoClient.Modes.FolderStore;
-using Logging;
+using Logging;
 
 namespace AutoClient
 {
@@ -20,15 +19,6 @@ namespace AutoClient
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
-
-            if (!string.IsNullOrEmpty(config.FolderToStore))
-            {
-                FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
-            }
-            else
-            {
-                FolderWorkDispatcher = null!;
-            }
         }
 
         public Configuration Config { get; }
@@ -37,7 +27,6 @@ namespace AutoClient
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public CidRepo CidRepo { get; }
         public Performance Performance { get; }
-        public FolderWorkDispatcher FolderWorkDispatcher { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -1,165 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public class AutomaticPurchaser
-    {
-        private readonly ILog log;
-        private readonly ICodexInstance instance;
-        private readonly CodexNode codex;
-        private Task workerTask = Task.CompletedTask;
-        private App app => instance.App;
-
-        public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
-        {
-            this.log = log;
-            this.instance = instance;
-            this.codex = codex;
-        }
-
-        public void Start()
-        {
-            workerTask = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            workerTask.Wait();
-        }
-
-        private async Task Worker()
-        {
-            log.Log("Worker started.");
-            while (!app.Cts.Token.IsCancellationRequested)
-            {
-                try
-                {
-                    var pid = await StartNewPurchase();
-                    await WaitTillFinished(pid);
-                    await DownloadForeignCid();
-                }
-                catch (Exception ex)
-                {
-                    log.Error("Worker failed with: " + ex);
-                    await Task.Delay(TimeSpan.FromHours(6));
-                }
-            }
-        }
-
-        private async Task DownloadForeignCid()
-        {
-            var cid = app.CidRepo.GetForeignCid(instance.NodeId);
-            if (cid == null) return;
-
-            var size = app.CidRepo.GetSizeForCid(cid);
-            if (size == null) return;
-
-            var filename = Guid.NewGuid().ToString().ToLowerInvariant();
-            await codex.DownloadCid(filename, cid, size);
-
-            DeleteFile(filename);
-        }
-
-        private async Task<string> StartNewPurchase()
-        {
-            var file = await CreateFile();
-            try
-            {
-                var cid = await codex.UploadFile(file);
-                var response = await codex.RequestStorage(cid);
-                return response.PurchaseId;
-            }
-            finally
-            {
-                DeleteFile(file);
-            }
-        }
-
-        private async Task<string> CreateFile()
-        {
-            return await app.Generator.Generate();
-        }
-
-        private void DeleteFile(string file)
-        {
-            try
-            {
-                File.Delete(file);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error($"Failed to delete file '{file}': {exc}");
-            }
-        }
-
-        private async Task WaitTillFinished(string pid)
-        {
-            try
-            {
-                var emptyResponseTolerance = 10;
-                while (!app.Cts.Token.IsCancellationRequested)
-                {
-                    var purchase = await codex.GetStoragePurchase(pid);
-                    if (purchase == null)
-                    {
-                        await FixedShortDelay();
-                        emptyResponseTolerance--;
-                        if (emptyResponseTolerance == 0)
-                        {
-                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
-                            await ExpiryTimeDelay();
-                            return;
-                        }
-                        continue;
-                    }
-                    if (purchase.IsCancelled)
-                    {
-                        app.Performance.StorageContractCancelled();
-                        return;
-                    }
-                    if (purchase.IsError)
-                    {
-                        app.Performance.StorageContractErrored(purchase.Error);
-                        return;
-                    }
-                    if (purchase.IsFinished)
-                    {
-                        app.Performance.StorageContractFinished();
-                        return;
-                    }
-                    if (purchase.IsStarted)
-                    {
-                        app.Performance.StorageContractStarted();
-                        await FixedDurationDelay();
-                    }
-
-                    await FixedShortDelay();
-                }
-            }
-            catch (Exception ex)
-            {
-                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
-                await ExpiryTimeDelay();
-            }
-        }
-
-        private async Task FixedDurationDelay()
-        {
-            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task ExpiryTimeDelay()
-        {
-            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task FixedShortDelay()
-        {
-            await Task.Delay(15 * 1000, app.Cts.Token);
-        }
-    }
-}

Tools/AutoClient/CidRepo.cs

@@ -17,7 +17,6 @@
             lock (_lock)
             {
                 entries.Add(new CidEntry(nodeId, cid, knownSize));
-                if (entries.Count > 1000) entries.Clear();
             }
         }
 

Tools/AutoClient/CodexInstance.cs

@@ -1,161 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Nethereum.Model;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public interface ICodexInstance
-    {
-        string NodeId { get; }
-        App App { get; }
-        CodexApi Codex { get; }
-        HttpClient Client { get; }
-        Address Address { get; }
-    }
-
-    public class CodexInstance : ICodexInstance
-    {
-        public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
-        {
-            App = app;
-            Codex = codex;
-            Client = client;
-            Address = address;
-            NodeId = Guid.NewGuid().ToString();
-        }
-
-        public string NodeId { get; }
-        public App App { get; }
-        public CodexApi Codex { get; }
-        public HttpClient Client { get; }
-        public Address Address { get; }
-    }
-
-    public class CodexNode
-    {
-        private readonly App app;
-        private readonly ICodexInstance codex;
-
-        public CodexNode(App app, ICodexInstance instance)
-        {
-            this.app = app;
-            codex = instance;
-        }
-
-        public async Task DownloadCid(string filename, string cid, long? size)
-        {
-            try
-            {
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                using var fileStream = File.OpenWrite(filename);
-                var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
-                fileResponse.Stream.CopyTo(fileStream);
-                var time = sw.Elapsed;
-                app.Performance.DownloadSuccessful(size, time);
-            }
-            catch (Exception ex)
-            {
-                app.Performance.DownloadFailed(ex);
-            }
-        }
-
-        public async Task<ContentId> UploadFile(string filename)
-        {
-            using var fileStream = File.OpenRead(filename);
-            try
-            {
-                var info = new FileInfo(filename);
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream, filename);
-                var time = sw.Elapsed;
-                app.Performance.UploadSuccessful(info.Length, time);
-                app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
-                return cid;
-            }
-            catch (Exception exc)
-            {
-                app.Performance.UploadFailed(exc);
-                throw;
-            }
-        }
-
-        public async Task<RequestStorageResult> RequestStorage(ContentId cid)
-        {
-            app.Log.Debug("Requesting storage for " + cid.Id);
-            var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
-            {
-                Collateral = app.Config.RequiredCollateral.ToString(),
-                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = app.Config.NumHosts,
-                Reward = app.Config.Price.ToString(),
-                ProofProbability = "15",
-                Tolerance = app.Config.HostTolerance
-            }, app.Cts.Token);
-
-            app.Log.Debug("Purchase ID: " + result);
-
-            var encoded = await GetEncodedCid(result);
-            app.CidRepo.AddEncoded(cid.Id, encoded);
-
-            return new RequestStorageResult(result, new ContentId(encoded));
-        }
-
-        public class RequestStorageResult
-        {
-            public RequestStorageResult(string purchaseId, ContentId encodedCid)
-            {
-                PurchaseId = purchaseId;
-                EncodedCid = encodedCid;
-            }
-
-            public string PurchaseId { get; }
-            public ContentId EncodedCid { get; }
-
-            public override string ToString()
-            {
-                return $"{PurchaseId} (cid: {EncodedCid})";
-            }
-        }
-
-        public async Task<StoragePurchase?> GetStoragePurchase(string pid)
-        {
-            // openapi still don't match code.
-            var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
-            if (string.IsNullOrEmpty(str)) return null;
-            return JsonConvert.DeserializeObject<StoragePurchase>(str);
-        }
-
-        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
-        {
-            app.Log.Debug($"Uploading file...");
-            var response = await codex.Codex.UploadAsync(
-                content_type: "application/octet-stream",
-                content_disposition: $"attachment; filename=\"{filename}\"",
-                fileStream, app.Cts.Token);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
-
-            app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        private async Task<string> GetEncodedCid(string pid)
-        {
-            try
-            {
-                var sp = (await GetStoragePurchase(pid))!;
-                return sp.Request.Content.Cid;
-            }
-            catch (Exception ex)
-            {
-                app.Log.Error(ex.ToString());
-                throw;
-            }
-        }
-    }
-}

Tools/AutoClient/CodexUser.cs

@@ -1,23 +1,32 @@
-using Logging;
+using CodexOpenApi;
+using Logging;
+using Utils;
 
-namespace AutoClient.Modes
+namespace AutoClient
 {
-    public class PurchasingMode : IMode
+    public class CodexUser
     {
-        private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
         private readonly App app;
+        private readonly CodexApi codex;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly List<Purchaser> purchasers = new List<Purchaser>();
         private Task starterTask = Task.CompletedTask;
+        private readonly string nodeId = Guid.NewGuid().ToString();
 
-        public PurchasingMode(App app)
+        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
         {
             this.app = app;
+            this.codex = codex;
+            this.client = client;
+            this.address = address;
         }
 
-        public void Start(ICodexInstance instance, int index)
+        public void Start(int index)
         {
             for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
             {
-                purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
+                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
             }
 
             var delayPerPurchaser =

Tools/AutoClient/Configuration.cs

@@ -5,8 +5,7 @@ namespace AutoClient
     public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
-        public string CodexEndpoints { get; set; } =
-            "http://localhost:8080";
+        public string CodexEndpoints { get; set; } = "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -15,22 +14,19 @@ namespace AutoClient
         public int NumConcurrentPurchases { get; set; } = 10;
 
         [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
-        public int ContractDurationMinutes { get; set; } =
-            60 * 24 * 6; // 6 days
-            //60 * 6; 6 hours
-        // Cluster nodes configured for max 7-day storage.
+        public int ContractDurationMinutes { get; set; } = 60 * 6;
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 60;
+        public int ContractExpiryMinutes { get; set; } = 15;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
-        public int NumHosts { get; set; } = 5;
+        public int NumHosts { get; set; } = 10;
 
         [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
-        public int HostTolerance { get; set; } = 1;
+        public int HostTolerance { get; set; } = 5;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
-        public int Price { get; set; } = 1000;
+        public int Price { get; set; } = 10;
 
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
@@ -38,9 +34,6 @@ namespace AutoClient
         [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
         public int FileSizeMb { get; set; } = 0;
 
-        [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
-        public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
-
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -1,58 +0,0 @@
-using static AutoClient.Modes.FolderStore.FileWorker;
-
-namespace AutoClient.Modes.FolderStore
-{
-    public class FileStatus : JsonBacked<WorkerStatus>
-    {
-        private readonly PurchaseInfo purchaseInfo;
-
-        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
-            : base(app, folder, filePath + ".json")
-        {
-            this.purchaseInfo = purchaseInfo;
-        }
-
-        public bool IsBusy()
-        {
-            if (!State.Purchases.Any()) return false;
-
-            return State.Purchases.Any(p =>
-                p.Submitted.HasValue &&
-                !p.Started.HasValue &&
-                !p.Expiry.HasValue &&
-                !p.Finish.HasValue &&
-                p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
-            );
-        }
-
-        public bool IsCurrentlyRunning()
-        {
-            if (!State.Purchases.Any()) return false;
-
-            return State.Purchases.Any(p =>
-                p.Submitted.HasValue &&
-                p.Started.HasValue &&
-                !p.Expiry.HasValue &&
-                !p.Finish.HasValue &&
-                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
-            );
-        }
-
-        public bool IsCurrentlyFailed()
-        {
-            if (!State.Purchases.Any()) return false;
-
-            var mostRecent = GetMostRecent();
-            if (mostRecent == null) return false;
-
-            return mostRecent.Expiry.HasValue;
-        }
-
-        protected WorkerPurchase? GetMostRecent()
-        {
-            if (!State.Purchases.Any()) return null;
-            var maxCreated = State.Purchases.Max(p => p.Created);
-            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
-        }
-    }
-}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -1,293 +0,0 @@
-using Logging;
-
-namespace AutoClient.Modes.FolderStore
-{
-    public class FileWorker : FileStatus
-    {
-        private readonly App app;
-        private readonly ILog log;
-        private readonly ICodexInstance instance;
-        private readonly PurchaseInfo purchaseInfo;
-        private readonly string sourceFilename;
-        private readonly Action onFileUploaded;
-        private readonly Action onNewPurchase;
-        private readonly CodexNode codex;
-
-        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
-            : base(app, folder, fileIndex.File + ".json", purchaseInfo)
-        {
-            this.app = app;
-            log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
-            this.instance = instance;
-            this.purchaseInfo = purchaseInfo;
-            sourceFilename = fileIndex.File;
-            if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
-            this.onFileUploaded = onFileUploaded;
-            this.onNewPurchase = onNewPurchase;
-            codex = new CodexNode(app, instance);
-        }
-
-        public int FailureCounter => State.FailureCounter;
-
-        protected override void OnNewState(WorkerStatus newState)
-        {
-            newState.LastUpdate = DateTime.MinValue;
-        }
-
-        public async Task Update()
-        {
-            try
-            {
-                if (IsCurrentlyRunning() && UpdatedRecently()) return;
-
-                Log($"Updating for '{sourceFilename}'...");
-                await EnsureRecentPurchase();
-                SaveState();
-                app.Log.Log("");
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error("Exception during fileworker update: " + exc);
-                State.Error = exc.ToString();
-                SaveState();
-                throw;
-            }
-        }
-
-        private bool UpdatedRecently()
-        {
-            var now = DateTime.UtcNow;
-            return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
-        }
-
-        private async Task<string> EnsureCid()
-        {
-            Log($"Checking CID...");
-
-            if (!string.IsNullOrEmpty(State.EncodedCid) &&
-                await DoesCidExistInNetwork(State.EncodedCid))
-            {
-                Log("Encoded-CID successfully found in the network.");
-                // TODO: Using the encoded CID currently would result in double-encoding of the dataset.
-                // See: https://github.com/codex-storage/nim-codex/issues/1005
-                // Always use the basic CID for now, even though we have to repeat the encoding.
-                // When using encoded CID works: return State.EncodedCid;
-            }
-
-            if (!string.IsNullOrEmpty(State.Cid) &&
-                await DoesCidExistInNetwork(State.Cid))
-            {
-                Log("Basic-CID successfully found in the network.");
-                return State.Cid;
-            }
-
-            if (string.IsNullOrEmpty(State.Cid))
-            {
-                Log("File was not previously uploaded.");
-            }
-
-            Log($"Uploading...");
-            var cid = await codex.UploadFile(sourceFilename);
-            onFileUploaded();
-            Log("Got Basic-CID: " + cid);
-            State.Cid = cid.Id;
-            SaveState();
-            return State.Cid;
-        }
-
-        private async Task<bool> DoesCidExistInNetwork(string cid)
-        {
-            try
-            {
-                // This should not take longer than a few seconds. If it does, cancel it.
-                var cts = new CancellationTokenSource();
-                var cancelTask = Task.Run(() =>
-                {
-                    Thread.Sleep(TimeSpan.FromSeconds(15));
-                    cts.Cancel();
-                });
-
-                var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
-                if (manifest == null) return false;
-            }
-            catch
-            {
-                return false;
-            }
-            return true;
-        }
-
-        private async Task EnsureRecentPurchase()
-        {
-            Log($"Checking recent purchase...");
-            var recent = GetMostRecent();
-            if (recent == null)
-            {
-                Log($"No recent purchase.");
-                await MakeNewPurchase();
-                return;
-            }
-
-            await UpdatePurchase(recent);
-
-            if (recent.Expiry.HasValue)
-            {
-                Log($"Purchase has failed or expired.");
-                await MakeNewPurchase();
-                State.FailureCounter++;
-                return;
-            }
-
-            if (recent.Finish.HasValue)
-            {
-                Log($"Purchase has finished.");
-                await MakeNewPurchase();
-                return;
-            }
-
-            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
-            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
-            {
-                Log($"Purchase is going to expire soon.");
-                await MakeNewPurchase();
-                return;
-            }
-
-            if (!recent.Submitted.HasValue)
-            {
-                Log($"Purchase is waiting to be submitted.");
-                return;
-            }
-
-            if (recent.Submitted.HasValue && !recent.Started.HasValue)
-            {
-                Log($"Purchase is submitted and waiting to start.");
-                return;
-            }
-
-            Log($"Purchase is running.");
-        }
-
-        private async Task UpdatePurchase(WorkerPurchase recent)
-        {
-            if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
-            var now = DateTime.UtcNow;
-
-            var purchase = await codex.GetStoragePurchase(recent.Pid);
-            if (purchase == null)
-            {
-                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
-                recent.Expiry = now;
-                return;
-            }
-
-            if (purchase.IsSubmitted)
-            {
-                if (!recent.Submitted.HasValue) recent.Submitted = now;
-            }
-            if (purchase.IsStarted)
-            {
-                if (!recent.Submitted.HasValue) recent.Submitted = now;
-                if (!recent.Started.HasValue) recent.Started = now;
-            }
-            if (purchase.IsCancelled)
-            {
-                if (!recent.Submitted.HasValue) recent.Submitted = now;
-                if (!recent.Expiry.HasValue) recent.Expiry = now;
-            }
-            if (purchase.IsError)
-            {
-                if (!recent.Submitted.HasValue) recent.Submitted = now;
-                if (!recent.Expiry.HasValue) recent.Expiry = now;
-            }
-            if (purchase.IsFinished)
-            {
-                if (!recent.Submitted.HasValue) recent.Submitted = now;
-                if (!recent.Started.HasValue) recent.Started = now;
-                if (!recent.Finish.HasValue) recent.Finish = now;
-            }
-            State.LastUpdate = now;
-            SaveState();
-        }
-
-        private async Task MakeNewPurchase()
-        {
-            var cid = await EnsureCid();
-            if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
-
-            Log($"Creating new purchase...");
-            var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
-            var purchaseId = response.PurchaseId;
-            var encodedCid = response.EncodedCid;
-            if (string.IsNullOrEmpty(purchaseId) ||
-                purchaseId == "Unable to encode manifest" ||
-                purchaseId == "Purchasing not available" ||
-                purchaseId == "Expiry required" ||
-                purchaseId == "Expiry needs to be in future" ||
-                purchaseId == "Expiry has to be before the request's end (now + duration)")
-            {
-                throw new InvalidOperationException(purchaseId);
-            }
-
-            var newPurchase = new WorkerPurchase
-            {
-                Created = DateTime.UtcNow,
-                Pid = purchaseId
-            };
-            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
-            State.EncodedCid = encodedCid.Id;
-            SaveState();
-            onNewPurchase();
-
-            Log($"New purchase created. PID: '{purchaseId}'.");
-            Log("Got Encoded-CID: " + encodedCid);
-            Log("Waiting for submit...");
-            Thread.Sleep(500);
-
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
-            while (DateTime.UtcNow < timeout)
-            {
-                Thread.Sleep(5000);
-                await UpdatePurchase(newPurchase);
-                if (newPurchase.Submitted.HasValue)
-                {
-                    Log("New purchase successfully submitted.");
-                    return;
-                }
-            }
-            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
-        }
-
-        private void Log(string msg)
-        {
-            log.Log(msg);
-        }
-
-        private string GetFileTag(FileIndex filename)
-        {
-            return $"({filename.Index.ToString("00000")}) ";
-        }
-
-        [Serializable]
-        public class WorkerStatus
-        {
-            public DateTime LastUpdate { get; set; }
-            public string Cid { get; set; } = string.Empty;
-            public string EncodedCid { get; set; } = string.Empty;
-            public int FailureCounter { get; set; } = 0;
-            public string Error { get; set; } = string.Empty;
-            public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
-        }
-
-        [Serializable]
-        public class WorkerPurchase
-        {
-            public string Pid { get; set; } = string.Empty;
-            public DateTime Created { get; set; }
-            public DateTime? Submitted { get; set; }
-            public DateTime? Started { get; set; }
-            public DateTime? Expiry { get; set; }
-            public DateTime? Finish { get; set; }
-        }
-    }
-}

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -1,70 +0,0 @@
-using Logging;
-
-namespace AutoClient.Modes.FolderStore
-{
-    public class FolderWorkDispatcher
-    {
-        private readonly string[] files = Array.Empty<string>();
-        private readonly ILog log;
-        private int index = 0;
-        private int busyCount = 0;
-
-        public FolderWorkDispatcher(ILog log, string folder)
-        {
-            var fs = Directory.GetFiles(folder);
-            var result = new List<string>();
-            foreach (var f in fs)
-            {
-                if (!f.ToLowerInvariant().Contains(".json"))
-                {
-                    var info = new FileInfo(f);
-                    if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
-                    {
-                        result.Add(f);
-                    }
-                }
-            }
-            files = result.ToArray();
-            this.log = log;
-        }
-
-        public FileIndex GetFileToCheck()
-        {
-            if (busyCount > 0)
-            {
-                log.Log("");
-                log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
-                log.Log("");
-                ResetIndex();
-                Thread.Sleep(TimeSpan.FromMinutes(1));
-            }
-
-            var file = new FileIndex(files[index], index);
-            index = (index + 1) % files.Length;
-            return file;
-        }
-
-        public void ResetIndex()
-        {
-            index = 0;
-            busyCount = 0;
-        }
-
-        public void WorkerIsBusy()
-        {
-            busyCount++;
-        }
-    }
-
-    public class FileIndex
-    {
-        public FileIndex(string file, int index)
-        {
-            File = file;
-            Index = index;
-        }
-
-        public string File { get; }
-        public int Index { get; }
-    }
-}

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -1,136 +0,0 @@
-using CodexOpenApi;
-using System.IO.Compression;
-using static AutoClient.Modes.FolderStore.FolderWorkOverview;
-
-namespace AutoClient.Modes.FolderStore
-{
-    public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
-    {
-        private const string OverviewFilename = "codex_folder_saver_overview.json";
-        private readonly App app;
-        private readonly PurchaseInfo purchaseInfo;
-
-        public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
-            : base(app, folder, Path.Combine(folder, OverviewFilename))
-        {
-            this.app = app;
-            this.purchaseInfo = purchaseInfo;
-        }
-
-        protected override void OnNewState(WorkMonitorStatus newState)
-        {
-            newState.LastOverviewUpdate = DateTime.MinValue;
-        }
-
-        public async Task Update(ICodexInstance instance)
-        {
-            var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
-
-            var total = 0;
-            var successful = 0;
-            var failed = 0;
-            foreach (var file in jsonFiles)
-            {
-                try
-                {
-                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
-                    total++;
-                    if (worker.IsCurrentlyRunning()) successful++;
-                    if (worker.IsCurrentlyFailed()) failed++;
-                }
-                catch (Exception exc)
-                {
-                    app.Log.Error("Exception in workoverview update: " + exc);
-                }
-            }
-
-            State.TotalFiles = total;
-            State.SuccessfulStored = successful;
-            State.StoreFailed = failed;
-            SaveState();
-
-            if (State.UncommitedChanges > 3)
-            {
-                State.UncommitedChanges = 0;
-                SaveState();
-
-                await CreateNewOverviewZip(jsonFiles, FilePath, instance);
-            }
-        }
-
-        public void MarkUncommitedChange()
-        {
-            State.UncommitedChanges++;
-            SaveState();
-        }
-
-        private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
-        {
-            Log("");
-            Log("");
-            Log("Creating new overview zipfile...");
-            var zipFilename = CreateZipFile(jsonFiles, filePath);
-
-            Log("Uploading to Codex...");
-            try
-            {
-                var codex = new CodexNode(app, instance);
-                var cid = await codex.UploadFile(zipFilename);
-                Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
-                Log("Requesting storage for it...");
-                var result = await codex.RequestStorage(cid);
-                Log("Storage requested. Purchase ID: " + result);
-
-                var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
-                File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
-                Log($">>> [{outFile}] has been updated. <<<");
-            }
-            catch (Exception exc)
-            {
-                Log("Failed to upload new overview zipfile: " + exc);
-            }
-            Log("");
-            Log("");
-        }
-
-        private string CreateZipFile(List<string> jsonFiles, string filePath)
-        {
-            var zipFilename = Guid.NewGuid().ToString() + ".zip";
-
-            using (var memoryStream = new MemoryStream())
-            {
-                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
-                {
-                    archive.CreateEntryFromFile(filePath, "overview.json");
-                    foreach (var file in jsonFiles)
-                    {
-                        archive.CreateEntryFromFile(file, Path.GetFileName(file));
-                    }
-                }
-
-                using (var fileStream = new FileStream(zipFilename, FileMode.Create))
-                {
-                    memoryStream.Seek(0, SeekOrigin.Begin);
-                    memoryStream.CopyTo(fileStream);
-                }
-            }
-            return zipFilename;
-        }
-
-        private void Log(string msg)
-        {
-            app.Log.Log(msg);
-        }
-
-        [Serializable]
-        public class WorkMonitorStatus
-        {
-            public int TotalFiles { get; set; }
-            public int SuccessfulStored { get; set; }
-            public int StoreFailed { get; set; }
-
-            public DateTime LastOverviewUpdate { get; set; }
-            public int UncommitedChanges { get; set; }
-        }
-    }
-}

Tools/AutoClient/Modes/FolderStore/JsonBacked.cs

@@ -1,58 +0,0 @@
-using Newtonsoft.Json;
-
-namespace AutoClient.Modes.FolderStore
-{
-    public abstract class JsonBacked<T> where T : new()
-    {
-        private readonly App app;
-
-        protected JsonBacked(App app, string folder, string filePath)
-        {
-            this.app = app;
-            Folder = folder;
-            FilePath = filePath;
-            LoadState();
-        }
-
-        private void LoadState()
-        {
-            try
-            {
-                if (!File.Exists(FilePath))
-                {
-                    State = new T();
-                    OnNewState(State);
-                    SaveState();
-                }
-                var text = File.ReadAllText(FilePath);
-                State = JsonConvert.DeserializeObject<T>(text)!;
-                if (State == null) throw new Exception("Didn't deserialize " + FilePath);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error("Failed to load state: " + exc);
-            }
-        }
-
-        protected string Folder { get; }
-        protected string FilePath { get; }
-        protected T State { get; private set; } = default!;
-
-        protected virtual void OnNewState(T newState)
-        {
-        }
-
-        protected void SaveState()
-        {
-            try
-            {
-                var json = JsonConvert.SerializeObject(State);
-                File.WriteAllText(FilePath, json);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error("Failed to save state: " + exc);
-            }
-        }
-    }
-}

Tools/AutoClient/Modes/FolderStore/PurchaseInfo.cs

@@ -1,18 +0,0 @@
-namespace AutoClient.Modes.FolderStore
-{
-    public class PurchaseInfo
-    {
-        public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
-        {
-            PurchaseDurationTotal = purchaseDurationTotal;
-            PurchaseDurationSafe = purchaseDurationSafe;
-
-            if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
-            if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
-            if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
-        }
-
-        public TimeSpan PurchaseDurationTotal { get; }
-        public TimeSpan PurchaseDurationSafe { get; }
-    }
-}

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -1,86 +0,0 @@
-using AutoClient.Modes.FolderStore;
-
-namespace AutoClient.Modes
-{
-    public class FolderStoreMode : IMode
-    {
-        private readonly App app;
-        private readonly string folder;
-        private readonly PurchaseInfo purchaseInfo;
-        private readonly CancellationTokenSource cts = new CancellationTokenSource();
-        private Task checkTask = Task.CompletedTask;
-
-        public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
-        {
-            this.app = app;
-            this.folder = folder;
-            this.purchaseInfo = purchaseInfo;
-        }
-
-        public void Start(ICodexInstance instance, int index)
-        {
-            checkTask = Task.Run(async () =>
-            {
-                try
-                {
-                    await RunChecker(instance);
-                }
-                catch (Exception ex)
-                {
-                    app.Log.Error("Exception in FolderStoreMode worker: " + ex);
-                    Environment.Exit(1);
-                }
-            });
-        }
-
-        private async Task RunChecker(ICodexInstance instance)
-        {
-            var i = 0;
-            while (!cts.IsCancellationRequested)
-            {
-                Thread.Sleep(2000);
-
-                var worker = await ProcessWorkItem(instance);
-                if (worker.FailureCounter > 5)
-                {
-                    throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
-                }
-                i++;
-
-                if (i > 5)
-                {
-                    i = 0;
-                    var overview = new FolderWorkOverview(app, purchaseInfo, folder);
-                    await overview.Update(instance);
-                }
-            }
-        }
-
-        private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
-        {
-            var file = app.FolderWorkDispatcher.GetFileToCheck();
-            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
-            await worker.Update();
-            if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
-            return worker;
-        }
-
-        private void OnFileUploaded()
-        {
-        }
-
-        private void OnNewPurchase()
-        {
-            app.FolderWorkDispatcher.ResetIndex();
-
-            var overview = new FolderWorkOverview(app, purchaseInfo, folder);
-            overview.MarkUncommitedChange();
-        }
-
-        public void Stop()
-        {
-            cts.Cancel();
-            checkTask.Wait();
-        }
-    }
-}

Tools/AutoClient/Modes/Mode.cs

@@ -1,14 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace AutoClient.Modes
-{
-    public interface IMode
-    {
-        void Start(ICodexInstance instance, int index);
-        void Stop();
-    }
-}

Tools/AutoClient/Performance.cs

@@ -16,13 +16,11 @@ namespace AutoClient
             Log($"Download failed: {ex}");
         }
 
-        public void DownloadSuccessful(long? size, TimeSpan time)
+        public void DownloadSuccessful(long size, TimeSpan time)
         {
-            if (!size.HasValue) return;
-
             long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
             if (milliseconds < 1) milliseconds = 1;
-            long bytesPerSecond = 1000 * (size.Value / milliseconds);
+            long bytesPerSecond = 1000 * (size / milliseconds);
             Log($"Download successful: {bytesPerSecond} bytes per second");
         }
 

Tools/AutoClient/Program.cs

@@ -1,14 +1,11 @@
 using ArgsUniform;
 using AutoClient;
-using AutoClient.Modes;
-using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;
 
 public class Program
 {
     private readonly App app;
-    private readonly List<IMode> modes = new List<IMode>();
 
     public Program(Configuration config)
     {
@@ -34,60 +31,36 @@ public class Program
 
     public async Task Run()
     {
-        var codexInstances = await CreateCodexInstances();
+        var codexUsers = await CreateUsers();
 
         var i = 0;
-        foreach (var cdx in codexInstances)
+        foreach (var user in codexUsers)
         {
-            var mode = CreateMode();
-            modes.Add(mode);
-
-            mode.Start(cdx, i);
+            user.Start(i);
             i++;
         }
 
         app.Cts.Token.WaitHandle.WaitOne();
 
-        foreach (var mode in modes) mode.Stop();
-        modes.Clear();
+        foreach (var user in codexUsers) user.Stop();
 
         app.Log.Log("Done");
     }
 
-    private IMode CreateMode()
-    {
-        if (!string.IsNullOrEmpty(app.Config.FolderToStore))
-        {
-            return CreateFolderStoreMode();
-        }
-
-        return new PurchasingMode(app);
-    }
-
-    private IMode CreateFolderStoreMode()
-    {
-        if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
-
-        return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
-            purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
-            purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
-        ));
-    }
-
-    private async Task<CodexInstance[]> CreateCodexInstances()
+    private async Task<CodexUser[]> CreateUsers()
     {
         var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
-        var result = new List<CodexInstance>();
+        var result = new List<CodexUser>();
 
         foreach (var e in endpointStrs)
         {
-            result.Add(await CreateCodexInstance(e));
+            result.Add(await CreateUser(e));
         }
 
         return result.ToArray();
     }
 
-    private async Task<CodexInstance> CreateCodexInstance(string endpoint)
+    private async Task<CodexUser> CreateUser(string endpoint)
     {
         var splitIndex = endpoint.LastIndexOf(':');
         var host = endpoint.Substring(0, splitIndex);
@@ -99,7 +72,6 @@ public class Program
         );
 
         var client = new HttpClient();
-        client.Timeout = TimeSpan.FromMinutes(60.0);
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
@@ -107,7 +79,7 @@ public class Program
         await CheckCodex(codex);
         app.Log.Log("OK");
 
-        return new CodexInstance(
+        return new CodexUser(
             app,
             codex,
             client,
@@ -133,4 +105,4 @@ public class Program
     {
         Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
     }
-}
+}

Tools/AutoClient/Purchaser.cs

@@ -0,0 +1,258 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class Purchaser
+    {
+        private readonly App app;
+        private readonly string nodeId;
+        private readonly ILog log;
+        private readonly HttpClient client;
+        private readonly Address address;
+        private readonly CodexApi codex;
+        private Task workerTask = Task.CompletedTask;
+
+        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
+        {
+            this.app = app;
+            this.nodeId = nodeId;
+            this.log = log;
+            this.client = client;
+            this.address = address;
+            this.codex = codex;
+        }
+
+        public void Start()
+        {
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
+        }
+
+        private async Task Worker()
+        {
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(nodeId);
+            if (cid == null) return;
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+                {
+                    using var fileStream = File.OpenWrite(filename);
+                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
+                    fileResponse.Stream.CopyTo(fileStream);
+                }
+                var time = sw.Elapsed;
+                File.Delete(filename);
+                app.Performance.DownloadSuccessful(size.Value, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
+            }
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            try
+            {
+                var cid = await UploadFile(file);
+                return await RequestStorage(cid);
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
+        private async Task<ContentId> UploadFile(string filename)
+        {
+            using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(nodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
+
+        private async Task<ContentId> UploadStream(FileStream fileStream)
+        {
+            log.Debug($"Uploading file...");
+            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            log.Debug($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> RequestStorage(ContentId cid)
+        {
+            log.Debug("Requesting storage for " + cid.Id);
+            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
+
+            log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
+
+            return result;
+        }
+
+        private async Task<string> GetEncodedCid(string pid)
+        {
+            try
+            {
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
+            }
+            catch (Exception ex)
+            {
+                log.Error(ex.ToString());
+                throw;
+            }
+        }
+
+        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (!app.Cts.Token.IsCancellationRequested)
+                {
+                    var purchase = await GetStoragePurchase(pid);
+                    if (purchase == null)
+                    {
+                        await FixedShortDelay();
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                        continue;
+                    }
+                    var status = purchase.State.ToLowerInvariant();
+                    if (status.Contains("cancel"))
+                    {
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (status.Contains("error"))
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (status.Contains("finished"))
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (status.Contains("started"))
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, app.Cts.Token);
+        }
+    }
+}

Tools/BiblioTech/CodexCidChecker.cs

@@ -1,6 +1,5 @@
 using CodexOpenApi;
 using IdentityModel.Client;
-using Logging;
 using Utils;
 
 namespace BiblioTech
@@ -9,13 +8,11 @@ namespace BiblioTech
     {
         private static readonly string nl = Environment.NewLine;
         private readonly Configuration config;
-        private readonly ILog log;
         private CodexApi? currentCodexNode;
 
-        public CodexCidChecker(Configuration config, ILog log)
+        public CodexCidChecker(Configuration config)
         {
             this.config = config;
-            this.log = log;
         }
 
         public async Task<CheckResponse> PerformCheck(string cid)
@@ -66,7 +63,7 @@ namespace BiblioTech
                 success: true,
                 title: $"Success: '{content.Cid}'",
                 error: "",
-                $"size: {content.Manifest.DatasetSize} bytes",
+                $"size: {content.Manifest.OriginalBytes} bytes",
                 $"blockSize: {content.Manifest.BlockSize} bytes",
                 $"protected: {content.Manifest.Protected}"
             );
@@ -153,7 +150,6 @@ namespace BiblioTech
             }
             catch (Exception e)
             {
-                log.Error(e.ToString());
                 return false;
             }
         }

Tools/BiblioTech/Program.cs

@@ -80,7 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checker = new CodexCidChecker(Config, Log);
+            var checker = new CodexCidChecker(Config);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();