Creates new overview after 3 new changes.

Use ETH_PRIVATE_KEY variable instead of deprecated PRIV_KEY

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -75,11 +75,12 @@ namespace CodexContractsPlugin.ChainMonitor
                 throw new Exception(msg);
             }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -63,10 +63,10 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
@@ -82,7 +82,20 @@ namespace CodexPlugin
 
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -261,4 +274,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:0.1.6-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -109,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -18,6 +18,7 @@ namespace CodexPlugin
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDatasetList LocalFiles();
@@ -138,6 +139,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +151,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
@@ -264,10 +271,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,16 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
+            };
+        }
+
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
             };
         }
 
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -136,47 +153,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -184,8 +199,20 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -245,4 +272,42 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -40,6 +40,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -90,6 +90,40 @@ components:
         cid:
           $ref: "#/components/schemas/Cid"
 
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
+
     DebugInfo:
       type: object
       properties:
@@ -104,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -306,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -319,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -404,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:
@@ -455,7 +526,7 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
-    get:
+    post:
       summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
       tags: [ Data ]
       operationId: downloadNetwork
@@ -844,4 +915,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";

Tests/CodexTests/BasicTests/AsyncProfiling.cs

@@ -0,0 +1,85 @@
+using NUnit.Framework;
+using MetricsPlugin;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class AsyncProfiling : CodexDistTest
+    {
+        [Test]
+        public void AsyncProfileMetricsPlz()
+        {
+            var node = StartCodex(s => s.EnableMetrics());
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single();
+
+            var file = GenerateTestFile(100.MB());
+            node.UploadFile(file);
+
+            Thread.Sleep(10000);
+
+            var profilerMetrics = new AsyncProfileMetrics(metrics.GetAllMetrics());
+
+            var log = GetTestLog();
+            log.Log($"{nameof(profilerMetrics.CallCount)} = {profilerMetrics.CallCount.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTime)} = {profilerMetrics.ExecTime.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTimeWithChildren)} = {profilerMetrics.ExecTimeWithChildren.Highest()}");
+            log.Log($"{nameof(profilerMetrics.SingleExecTimeMax)} = {profilerMetrics.SingleExecTimeMax.Highest()}");
+            log.Log($"{nameof(profilerMetrics.WallTime)} = {profilerMetrics.WallTime.Highest()}");
+        }
+    }
+
+    public class AsyncProfileMetrics
+    {
+        public AsyncProfileMetrics(Metrics metrics)
+        {
+            CallCount = CreateMetric(metrics, "chronos_call_count_total");
+            ExecTime = CreateMetric(metrics, "chronos_exec_time_total");
+            ExecTimeWithChildren = CreateMetric(metrics, "chronos_exec_time_with_children_total");
+            SingleExecTimeMax = CreateMetric(metrics, "chronos_single_exec_time_max");
+            WallTime = CreateMetric(metrics, "chronos_wall_time_total");
+        }
+
+        public AsyncProfileMetric CallCount { get; }
+        public AsyncProfileMetric ExecTime { get; }
+        public AsyncProfileMetric ExecTimeWithChildren { get; }
+        public AsyncProfileMetric SingleExecTimeMax { get; }
+        public AsyncProfileMetric WallTime { get; }
+
+        private static AsyncProfileMetric CreateMetric(Metrics metrics, string name)
+        {
+            var sets = metrics.Sets.Where(s => s.Name == name).ToArray();
+            return new AsyncProfileMetric(sets);
+        }
+    }
+
+    public class AsyncProfileMetric
+    {
+        private readonly MetricsSet[] metricsSets;
+
+        public AsyncProfileMetric(MetricsSet[] metricsSets)
+        {
+            this.metricsSets = metricsSets;
+        }
+
+        public MetricsSet Highest()
+        {
+            MetricsSet? result = null;
+            var highest = double.MinValue;
+            foreach (var metric in metricsSets)
+            {
+                foreach (var value in metric.Values)
+                {
+                    if (value.Value > highest)
+                    {
+                        highest = value.Value;
+                        result = metric;
+                    }
+                }
+            }
+
+            if (result == null) throw new Exception("None were highest");
+            return result;
+        }
+    }
+}

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -36,7 +36,7 @@ namespace CodexTests.BasicTests
             var primary2 = group2[0];
             var secondary2 = group2[1];
 
-            var metrics = Ci.GetMetricsFor(primary, primary2);
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(10), primary, primary2);
 
             primary.ConnectToPeer(secondary);
             primary2.ConnectToPeer(secondary2);

Tools/AutoClient/App.cs

@@ -1,4 +1,5 @@
-using Logging;
+using AutoClient.Modes.FolderStore;
+using Logging;
 
 namespace AutoClient
 {
@@ -19,6 +20,15 @@ namespace AutoClient
                 new FileLog(Path.Combine(config.LogPath, "performance")),
                 new ConsoleLog()
             ));
+
+            if (!string.IsNullOrEmpty(config.FolderToStore))
+            {
+                FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
+            }
+            else
+            {
+                FolderWorkDispatcher = null!;
+            }
         }
 
         public Configuration Config { get; }
@@ -27,6 +37,7 @@ namespace AutoClient
         public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
         public CidRepo CidRepo { get; }
         public Performance Performance { get; }
+        public FolderWorkDispatcher FolderWorkDispatcher { get; }
 
         private IFileGenerator CreateGenerator()
         {

Tools/AutoClient/AutomaticPurchaser.cs

@@ -0,0 +1,165 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public class AutomaticPurchaser
+    {
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly CodexNode codex;
+        private Task workerTask = Task.CompletedTask;
+        private App app => instance.App;
+
+        public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex)
+        {
+            this.log = log;
+            this.instance = instance;
+            this.codex = codex;
+        }
+
+        public void Start()
+        {
+            workerTask = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            workerTask.Wait();
+        }
+
+        private async Task Worker()
+        {
+            log.Log("Worker started.");
+            while (!app.Cts.Token.IsCancellationRequested)
+            {
+                try
+                {
+                    var pid = await StartNewPurchase();
+                    await WaitTillFinished(pid);
+                    await DownloadForeignCid();
+                }
+                catch (Exception ex)
+                {
+                    log.Error("Worker failed with: " + ex);
+                    await Task.Delay(TimeSpan.FromHours(6));
+                }
+            }
+        }
+
+        private async Task DownloadForeignCid()
+        {
+            var cid = app.CidRepo.GetForeignCid(instance.NodeId);
+            if (cid == null) return;
+
+            var size = app.CidRepo.GetSizeForCid(cid);
+            if (size == null) return;
+
+            var filename = Guid.NewGuid().ToString().ToLowerInvariant();
+            await codex.DownloadCid(filename, cid, size);
+
+            DeleteFile(filename);
+        }
+
+        private async Task<string> StartNewPurchase()
+        {
+            var file = await CreateFile();
+            try
+            {
+                var cid = await codex.UploadFile(file);
+                var response = await codex.RequestStorage(cid);
+                return response.PurchaseId;
+            }
+            finally
+            {
+                DeleteFile(file);
+            }
+        }
+
+        private async Task<string> CreateFile()
+        {
+            return await app.Generator.Generate();
+        }
+
+        private void DeleteFile(string file)
+        {
+            try
+            {
+                File.Delete(file);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error($"Failed to delete file '{file}': {exc}");
+            }
+        }
+
+        private async Task WaitTillFinished(string pid)
+        {
+            try
+            {
+                var emptyResponseTolerance = 10;
+                while (!app.Cts.Token.IsCancellationRequested)
+                {
+                    var purchase = await codex.GetStoragePurchase(pid);
+                    if (purchase == null)
+                    {
+                        await FixedShortDelay();
+                        emptyResponseTolerance--;
+                        if (emptyResponseTolerance == 0)
+                        {
+                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
+                            await ExpiryTimeDelay();
+                            return;
+                        }
+                        continue;
+                    }
+                    if (purchase.IsCancelled)
+                    {
+                        app.Performance.StorageContractCancelled();
+                        return;
+                    }
+                    if (purchase.IsError)
+                    {
+                        app.Performance.StorageContractErrored(purchase.Error);
+                        return;
+                    }
+                    if (purchase.IsFinished)
+                    {
+                        app.Performance.StorageContractFinished();
+                        return;
+                    }
+                    if (purchase.IsStarted)
+                    {
+                        app.Performance.StorageContractStarted();
+                        await FixedDurationDelay();
+                    }
+
+                    await FixedShortDelay();
+                }
+            }
+            catch (Exception ex)
+            {
+                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
+                await ExpiryTimeDelay();
+            }
+        }
+
+        private async Task FixedDurationDelay()
+        {
+            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task ExpiryTimeDelay()
+        {
+            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
+        }
+
+        private async Task FixedShortDelay()
+        {
+            await Task.Delay(15 * 1000, app.Cts.Token);
+        }
+    }
+}

Tools/AutoClient/CidRepo.cs

@@ -17,6 +17,7 @@
             lock (_lock)
             {
                 entries.Add(new CidEntry(nodeId, cid, knownSize));
+                if (entries.Count > 1000) entries.Clear();
             }
         }
 

Tools/AutoClient/CodexInstance.cs

@@ -0,0 +1,161 @@
+using CodexOpenApi;
+using CodexPlugin;
+using Logging;
+using Nethereum.Model;
+using Newtonsoft.Json;
+using Utils;
+
+namespace AutoClient
+{
+    public interface ICodexInstance
+    {
+        string NodeId { get; }
+        App App { get; }
+        CodexApi Codex { get; }
+        HttpClient Client { get; }
+        Address Address { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(App app, CodexApi codex, HttpClient client, Address address)
+        {
+            App = app;
+            Codex = codex;
+            Client = client;
+            Address = address;
+            NodeId = Guid.NewGuid().ToString();
+        }
+
+        public string NodeId { get; }
+        public App App { get; }
+        public CodexApi Codex { get; }
+        public HttpClient Client { get; }
+        public Address Address { get; }
+    }
+
+    public class CodexNode
+    {
+        private readonly App app;
+        private readonly ICodexInstance codex;
+
+        public CodexNode(App app, ICodexInstance instance)
+        {
+            this.app = app;
+            codex = instance;
+        }
+
+        public async Task DownloadCid(string filename, string cid, long? size)
+        {
+            try
+            {
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                using var fileStream = File.OpenWrite(filename);
+                var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid);
+                fileResponse.Stream.CopyTo(fileStream);
+                var time = sw.Elapsed;
+                app.Performance.DownloadSuccessful(size, time);
+            }
+            catch (Exception ex)
+            {
+                app.Performance.DownloadFailed(ex);
+            }
+        }
+
+        public async Task<ContentId> UploadFile(string filename)
+        {
+            using var fileStream = File.OpenRead(filename);
+            try
+            {
+                var info = new FileInfo(filename);
+                var sw = System.Diagnostics.Stopwatch.StartNew();
+                var cid = await UploadStream(fileStream, filename);
+                var time = sw.Elapsed;
+                app.Performance.UploadSuccessful(info.Length, time);
+                app.CidRepo.Add(codex.NodeId, cid.Id, info.Length);
+                return cid;
+            }
+            catch (Exception exc)
+            {
+                app.Performance.UploadFailed(exc);
+                throw;
+            }
+        }
+
+        public async Task<RequestStorageResult> RequestStorage(ContentId cid)
+        {
+            app.Log.Debug("Requesting storage for " + cid.Id);
+            var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
+            {
+                Collateral = app.Config.RequiredCollateral.ToString(),
+                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
+                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
+                Nodes = app.Config.NumHosts,
+                Reward = app.Config.Price.ToString(),
+                ProofProbability = "15",
+                Tolerance = app.Config.HostTolerance
+            }, app.Cts.Token);
+
+            app.Log.Debug("Purchase ID: " + result);
+
+            var encoded = await GetEncodedCid(result);
+            app.CidRepo.AddEncoded(cid.Id, encoded);
+
+            return new RequestStorageResult(result, new ContentId(encoded));
+        }
+
+        public class RequestStorageResult
+        {
+            public RequestStorageResult(string purchaseId, ContentId encodedCid)
+            {
+                PurchaseId = purchaseId;
+                EncodedCid = encodedCid;
+            }
+
+            public string PurchaseId { get; }
+            public ContentId EncodedCid { get; }
+
+            public override string ToString()
+            {
+                return $"{PurchaseId} (cid: {EncodedCid})";
+            }
+        }
+
+        public async Task<StoragePurchase?> GetStoragePurchase(string pid)
+        {
+            // openapi still don't match code.
+            var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}");
+            if (string.IsNullOrEmpty(str)) return null;
+            return JsonConvert.DeserializeObject<StoragePurchase>(str);
+        }
+
+        private async Task<ContentId> UploadStream(FileStream fileStream, string filename)
+        {
+            app.Log.Debug($"Uploading file...");
+            var response = await codex.Codex.UploadAsync(
+                content_type: "application/octet-stream",
+                content_disposition: $"attachment; filename=\"{filename}\"",
+                fileStream, app.Cts.Token);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
+
+            app.Log.Debug($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        private async Task<string> GetEncodedCid(string pid)
+        {
+            try
+            {
+                var sp = (await GetStoragePurchase(pid))!;
+                return sp.Request.Content.Cid;
+            }
+            catch (Exception ex)
+            {
+                app.Log.Error(ex.ToString());
+                throw;
+            }
+        }
+    }
+}

Tools/AutoClient/Configuration.cs

@@ -5,7 +5,8 @@ namespace AutoClient
     public class Configuration
     {
         [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")]
-        public string CodexEndpoints { get; set; } = "http://localhost:8080";
+        public string CodexEndpoints { get; set; } =
+            "http://localhost:8080";
 
         [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")]
         public string DataPath { get; set; } = "datapath";
@@ -14,19 +15,22 @@ namespace AutoClient
         public int NumConcurrentPurchases { get; set; } = 10;
 
         [Uniform("contract-duration", "cd", "CONTRACTDURATION", false, "contract duration in minutes. (default 6 hours)")]
-        public int ContractDurationMinutes { get; set; } = 60 * 6;
+        public int ContractDurationMinutes { get; set; } =
+            60 * 24 * 6; // 6 days
+            //60 * 6; 6 hours
+        // Cluster nodes configured for max 7-day storage.
 
         [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")]
-        public int ContractExpiryMinutes { get; set; } = 15;
+        public int ContractExpiryMinutes { get; set; } = 60;
 
         [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")]
-        public int NumHosts { get; set; } = 10;
+        public int NumHosts { get; set; } = 5;
 
         [Uniform("num-hosts-tolerance", "nt", "NUMTOL", false, "Number of host tolerance for contract. (default 5)")]
-        public int HostTolerance { get; set; } = 5;
+        public int HostTolerance { get; set; } = 1;
 
         [Uniform("price","p", "PRICE", false, "Price of contract. (default 10)")]
-        public int Price { get; set; } = 10;
+        public int Price { get; set; } = 1000;
 
         [Uniform("collateral", "c", "COLLATERAL", false, "Required collateral. (default 1)")]
         public int RequiredCollateral { get; set; } = 1;
@@ -34,6 +38,9 @@ namespace AutoClient
         [Uniform("filesizemb", "smb", "FILESIZEMB", false, "When greater than zero, size of file generated and uploaded. When zero, random images are used instead.")]
         public int FileSizeMb { get; set; } = 0;
 
+        [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")]
+        public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles";
+
         public string LogPath
         {
             get

Tools/AutoClient/Modes/FolderStore/FileStatus.cs

@@ -0,0 +1,58 @@
+using static AutoClient.Modes.FolderStore.FileWorker;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileStatus : JsonBacked<WorkerStatus>
+    {
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
+            : base(app, folder, filePath + ".json")
+        {
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public bool IsBusy()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                !p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyRunning()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            return State.Purchases.Any(p =>
+                p.Submitted.HasValue &&
+                p.Started.HasValue &&
+                !p.Expiry.HasValue &&
+                !p.Finish.HasValue &&
+                p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
+            );
+        }
+
+        public bool IsCurrentlyFailed()
+        {
+            if (!State.Purchases.Any()) return false;
+
+            var mostRecent = GetMostRecent();
+            if (mostRecent == null) return false;
+
+            return mostRecent.Expiry.HasValue;
+        }
+
+        protected WorkerPurchase? GetMostRecent()
+        {
+            if (!State.Purchases.Any()) return null;
+            var maxCreated = State.Purchases.Max(p => p.Created);
+            return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FileWorker.cs

@@ -0,0 +1,293 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FileWorker : FileStatus
+    {
+        private readonly App app;
+        private readonly ILog log;
+        private readonly ICodexInstance instance;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly string sourceFilename;
+        private readonly Action onFileUploaded;
+        private readonly Action onNewPurchase;
+        private readonly CodexNode codex;
+
+        public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase)
+            : base(app, folder, fileIndex.File + ".json", purchaseInfo)
+        {
+            this.app = app;
+            log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
+            this.instance = instance;
+            this.purchaseInfo = purchaseInfo;
+            sourceFilename = fileIndex.File;
+            if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
+            this.onFileUploaded = onFileUploaded;
+            this.onNewPurchase = onNewPurchase;
+            codex = new CodexNode(app, instance);
+        }
+
+        public int FailureCounter => State.FailureCounter;
+
+        protected override void OnNewState(WorkerStatus newState)
+        {
+            newState.LastUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update()
+        {
+            try
+            {
+                if (IsCurrentlyRunning() && UpdatedRecently()) return;
+
+                Log($"Updating for '{sourceFilename}'...");
+                await EnsureRecentPurchase();
+                SaveState();
+                app.Log.Log("");
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Exception during fileworker update: " + exc);
+                State.Error = exc.ToString();
+                SaveState();
+                throw;
+            }
+        }
+
+        private bool UpdatedRecently()
+        {
+            var now = DateTime.UtcNow;
+            return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
+        }
+
+        private async Task<string> EnsureCid()
+        {
+            Log($"Checking CID...");
+
+            if (!string.IsNullOrEmpty(State.EncodedCid) &&
+                await DoesCidExistInNetwork(State.EncodedCid))
+            {
+                Log("Encoded-CID successfully found in the network.");
+                // TODO: Using the encoded CID currently would result in double-encoding of the dataset.
+                // See: https://github.com/codex-storage/nim-codex/issues/1005
+                // Always use the basic CID for now, even though we have to repeat the encoding.
+                // When using encoded CID works: return State.EncodedCid;
+            }
+
+            if (!string.IsNullOrEmpty(State.Cid) &&
+                await DoesCidExistInNetwork(State.Cid))
+            {
+                Log("Basic-CID successfully found in the network.");
+                return State.Cid;
+            }
+
+            if (string.IsNullOrEmpty(State.Cid))
+            {
+                Log("File was not previously uploaded.");
+            }
+
+            Log($"Uploading...");
+            var cid = await codex.UploadFile(sourceFilename);
+            onFileUploaded();
+            Log("Got Basic-CID: " + cid);
+            State.Cid = cid.Id;
+            SaveState();
+            return State.Cid;
+        }
+
+        private async Task<bool> DoesCidExistInNetwork(string cid)
+        {
+            try
+            {
+                // This should not take longer than a few seconds. If it does, cancel it.
+                var cts = new CancellationTokenSource();
+                var cancelTask = Task.Run(() =>
+                {
+                    Thread.Sleep(TimeSpan.FromSeconds(15));
+                    cts.Cancel();
+                });
+
+                var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token);
+                if (manifest == null) return false;
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+        private async Task EnsureRecentPurchase()
+        {
+            Log($"Checking recent purchase...");
+            var recent = GetMostRecent();
+            if (recent == null)
+            {
+                Log($"No recent purchase.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            await UpdatePurchase(recent);
+
+            if (recent.Expiry.HasValue)
+            {
+                Log($"Purchase has failed or expired.");
+                await MakeNewPurchase();
+                State.FailureCounter++;
+                return;
+            }
+
+            if (recent.Finish.HasValue)
+            {
+                Log($"Purchase has finished.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
+            if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
+            {
+                Log($"Purchase is going to expire soon.");
+                await MakeNewPurchase();
+                return;
+            }
+
+            if (!recent.Submitted.HasValue)
+            {
+                Log($"Purchase is waiting to be submitted.");
+                return;
+            }
+
+            if (recent.Submitted.HasValue && !recent.Started.HasValue)
+            {
+                Log($"Purchase is submitted and waiting to start.");
+                return;
+            }
+
+            Log($"Purchase is running.");
+        }
+
+        private async Task UpdatePurchase(WorkerPurchase recent)
+        {
+            if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
+            var now = DateTime.UtcNow;
+
+            var purchase = await codex.GetStoragePurchase(recent.Pid);
+            if (purchase == null)
+            {
+                Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
+                recent.Expiry = now;
+                return;
+            }
+
+            if (purchase.IsSubmitted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+            }
+            if (purchase.IsStarted)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+            }
+            if (purchase.IsCancelled)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsError)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Expiry.HasValue) recent.Expiry = now;
+            }
+            if (purchase.IsFinished)
+            {
+                if (!recent.Submitted.HasValue) recent.Submitted = now;
+                if (!recent.Started.HasValue) recent.Started = now;
+                if (!recent.Finish.HasValue) recent.Finish = now;
+            }
+            State.LastUpdate = now;
+            SaveState();
+        }
+
+        private async Task MakeNewPurchase()
+        {
+            var cid = await EnsureCid();
+            if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
+
+            Log($"Creating new purchase...");
+            var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid));
+            var purchaseId = response.PurchaseId;
+            var encodedCid = response.EncodedCid;
+            if (string.IsNullOrEmpty(purchaseId) ||
+                purchaseId == "Unable to encode manifest" ||
+                purchaseId == "Purchasing not available" ||
+                purchaseId == "Expiry required" ||
+                purchaseId == "Expiry needs to be in future" ||
+                purchaseId == "Expiry has to be before the request's end (now + duration)")
+            {
+                throw new InvalidOperationException(purchaseId);
+            }
+
+            var newPurchase = new WorkerPurchase
+            {
+                Created = DateTime.UtcNow,
+                Pid = purchaseId
+            };
+            State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
+            State.EncodedCid = encodedCid.Id;
+            SaveState();
+            onNewPurchase();
+
+            Log($"New purchase created. PID: '{purchaseId}'.");
+            Log("Got Encoded-CID: " + encodedCid);
+            Log("Waiting for submit...");
+            Thread.Sleep(500);
+
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (DateTime.UtcNow < timeout)
+            {
+                Thread.Sleep(5000);
+                await UpdatePurchase(newPurchase);
+                if (newPurchase.Submitted.HasValue)
+                {
+                    Log("New purchase successfully submitted.");
+                    return;
+                }
+            }
+            Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
+        private string GetFileTag(FileIndex filename)
+        {
+            return $"({filename.Index.ToString("00000")}) ";
+        }
+
+        [Serializable]
+        public class WorkerStatus
+        {
+            public DateTime LastUpdate { get; set; }
+            public string Cid { get; set; } = string.Empty;
+            public string EncodedCid { get; set; } = string.Empty;
+            public int FailureCounter { get; set; } = 0;
+            public string Error { get; set; } = string.Empty;
+            public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
+        }
+
+        [Serializable]
+        public class WorkerPurchase
+        {
+            public string Pid { get; set; } = string.Empty;
+            public DateTime Created { get; set; }
+            public DateTime? Submitted { get; set; }
+            public DateTime? Started { get; set; }
+            public DateTime? Expiry { get; set; }
+            public DateTime? Finish { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs

@@ -0,0 +1,70 @@
+using Logging;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkDispatcher
+    {
+        private readonly string[] files = Array.Empty<string>();
+        private readonly ILog log;
+        private int index = 0;
+        private int busyCount = 0;
+
+        public FolderWorkDispatcher(ILog log, string folder)
+        {
+            var fs = Directory.GetFiles(folder);
+            var result = new List<string>();
+            foreach (var f in fs)
+            {
+                if (!f.ToLowerInvariant().Contains(".json"))
+                {
+                    var info = new FileInfo(f);
+                    if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
+                    {
+                        result.Add(f);
+                    }
+                }
+            }
+            files = result.ToArray();
+            this.log = log;
+        }
+
+        public FileIndex GetFileToCheck()
+        {
+            if (busyCount > 0)
+            {
+                log.Log("");
+                log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
+                log.Log("");
+                ResetIndex();
+                Thread.Sleep(TimeSpan.FromMinutes(1));
+            }
+
+            var file = new FileIndex(files[index], index);
+            index = (index + 1) % files.Length;
+            return file;
+        }
+
+        public void ResetIndex()
+        {
+            index = 0;
+            busyCount = 0;
+        }
+
+        public void WorkerIsBusy()
+        {
+            busyCount++;
+        }
+    }
+
+    public class FileIndex
+    {
+        public FileIndex(string file, int index)
+        {
+            File = file;
+            Index = index;
+        }
+
+        public string File { get; }
+        public int Index { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs

@@ -0,0 +1,136 @@
+using CodexOpenApi;
+using System.IO.Compression;
+using static AutoClient.Modes.FolderStore.FolderWorkOverview;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
+    {
+        private const string OverviewFilename = "codex_folder_saver_overview.json";
+        private readonly App app;
+        private readonly PurchaseInfo purchaseInfo;
+
+        public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
+            : base(app, folder, Path.Combine(folder, OverviewFilename))
+        {
+            this.app = app;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        protected override void OnNewState(WorkMonitorStatus newState)
+        {
+            newState.LastOverviewUpdate = DateTime.MinValue;
+        }
+
+        public async Task Update(ICodexInstance instance)
+        {
+            var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
+
+            var total = 0;
+            var successful = 0;
+            var failed = 0;
+            foreach (var file in jsonFiles)
+            {
+                try
+                {
+                    var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
+                    total++;
+                    if (worker.IsCurrentlyRunning()) successful++;
+                    if (worker.IsCurrentlyFailed()) failed++;
+                }
+                catch (Exception exc)
+                {
+                    app.Log.Error("Exception in workoverview update: " + exc);
+                }
+            }
+
+            State.TotalFiles = total;
+            State.SuccessfulStored = successful;
+            State.StoreFailed = failed;
+            SaveState();
+
+            if (State.UncommitedChanges > 3)
+            {
+                State.UncommitedChanges = 0;
+                SaveState();
+
+                await CreateNewOverviewZip(jsonFiles, FilePath, instance);
+            }
+        }
+
+        public void MarkUncommitedChange()
+        {
+            State.UncommitedChanges++;
+            SaveState();
+        }
+
+        private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
+        {
+            Log("");
+            Log("");
+            Log("Creating new overview zipfile...");
+            var zipFilename = CreateZipFile(jsonFiles, filePath);
+
+            Log("Uploading to Codex...");
+            try
+            {
+                var codex = new CodexNode(app, instance);
+                var cid = await codex.UploadFile(zipFilename);
+                Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
+                Log("Requesting storage for it...");
+                var result = await codex.RequestStorage(cid);
+                Log("Storage requested. Purchase ID: " + result);
+
+                var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
+                File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]);
+                Log($">>> [{outFile}] has been updated. <<<");
+            }
+            catch (Exception exc)
+            {
+                Log("Failed to upload new overview zipfile: " + exc);
+            }
+            Log("");
+            Log("");
+        }
+
+        private string CreateZipFile(List<string> jsonFiles, string filePath)
+        {
+            var zipFilename = Guid.NewGuid().ToString() + ".zip";
+
+            using (var memoryStream = new MemoryStream())
+            {
+                using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
+                {
+                    archive.CreateEntryFromFile(filePath, "overview.json");
+                    foreach (var file in jsonFiles)
+                    {
+                        archive.CreateEntryFromFile(file, Path.GetFileName(file));
+                    }
+                }
+
+                using (var fileStream = new FileStream(zipFilename, FileMode.Create))
+                {
+                    memoryStream.Seek(0, SeekOrigin.Begin);
+                    memoryStream.CopyTo(fileStream);
+                }
+            }
+            return zipFilename;
+        }
+
+        private void Log(string msg)
+        {
+            app.Log.Log(msg);
+        }
+
+        [Serializable]
+        public class WorkMonitorStatus
+        {
+            public int TotalFiles { get; set; }
+            public int SuccessfulStored { get; set; }
+            public int StoreFailed { get; set; }
+
+            public DateTime LastOverviewUpdate { get; set; }
+            public int UncommitedChanges { get; set; }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/JsonBacked.cs

@@ -0,0 +1,58 @@
+using Newtonsoft.Json;
+
+namespace AutoClient.Modes.FolderStore
+{
+    public abstract class JsonBacked<T> where T : new()
+    {
+        private readonly App app;
+
+        protected JsonBacked(App app, string folder, string filePath)
+        {
+            this.app = app;
+            Folder = folder;
+            FilePath = filePath;
+            LoadState();
+        }
+
+        private void LoadState()
+        {
+            try
+            {
+                if (!File.Exists(FilePath))
+                {
+                    State = new T();
+                    OnNewState(State);
+                    SaveState();
+                }
+                var text = File.ReadAllText(FilePath);
+                State = JsonConvert.DeserializeObject<T>(text)!;
+                if (State == null) throw new Exception("Didn't deserialize " + FilePath);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to load state: " + exc);
+            }
+        }
+
+        protected string Folder { get; }
+        protected string FilePath { get; }
+        protected T State { get; private set; } = default!;
+
+        protected virtual void OnNewState(T newState)
+        {
+        }
+
+        protected void SaveState()
+        {
+            try
+            {
+                var json = JsonConvert.SerializeObject(State);
+                File.WriteAllText(FilePath, json);
+            }
+            catch (Exception exc)
+            {
+                app.Log.Error("Failed to save state: " + exc);
+            }
+        }
+    }
+}

Tools/AutoClient/Modes/FolderStore/PurchaseInfo.cs

@@ -0,0 +1,18 @@
+namespace AutoClient.Modes.FolderStore
+{
+    public class PurchaseInfo
+    {
+        public PurchaseInfo(TimeSpan purchaseDurationTotal, TimeSpan purchaseDurationSafe)
+        {
+            PurchaseDurationTotal = purchaseDurationTotal;
+            PurchaseDurationSafe = purchaseDurationSafe;
+
+            if (PurchaseDurationTotal < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationTotal));
+            if (PurchaseDurationSafe < TimeSpan.Zero) throw new Exception(nameof(PurchaseDurationSafe));
+            if (PurchaseDurationTotal < PurchaseDurationSafe) throw new Exception("TotalDuration < SafeDuration");
+        }
+
+        public TimeSpan PurchaseDurationTotal { get; }
+        public TimeSpan PurchaseDurationSafe { get; }
+    }
+}

Tools/AutoClient/Modes/FolderStoreMode.cs

@@ -0,0 +1,86 @@
+using AutoClient.Modes.FolderStore;
+
+namespace AutoClient.Modes
+{
+    public class FolderStoreMode : IMode
+    {
+        private readonly App app;
+        private readonly string folder;
+        private readonly PurchaseInfo purchaseInfo;
+        private readonly CancellationTokenSource cts = new CancellationTokenSource();
+        private Task checkTask = Task.CompletedTask;
+
+        public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
+        {
+            this.app = app;
+            this.folder = folder;
+            this.purchaseInfo = purchaseInfo;
+        }
+
+        public void Start(ICodexInstance instance, int index)
+        {
+            checkTask = Task.Run(async () =>
+            {
+                try
+                {
+                    await RunChecker(instance);
+                }
+                catch (Exception ex)
+                {
+                    app.Log.Error("Exception in FolderStoreMode worker: " + ex);
+                    Environment.Exit(1);
+                }
+            });
+        }
+
+        private async Task RunChecker(ICodexInstance instance)
+        {
+            var i = 0;
+            while (!cts.IsCancellationRequested)
+            {
+                Thread.Sleep(2000);
+
+                var worker = await ProcessWorkItem(instance);
+                if (worker.FailureCounter > 5)
+                {
+                    throw new Exception("Worker has failure count > 5. Stopping AutoClient...");
+                }
+                i++;
+
+                if (i > 5)
+                {
+                    i = 0;
+                    var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+                    await overview.Update(instance);
+                }
+            }
+        }
+
+        private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
+        {
+            var file = app.FolderWorkDispatcher.GetFileToCheck();
+            var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
+            await worker.Update();
+            if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy();
+            return worker;
+        }
+
+        private void OnFileUploaded()
+        {
+        }
+
+        private void OnNewPurchase()
+        {
+            app.FolderWorkDispatcher.ResetIndex();
+
+            var overview = new FolderWorkOverview(app, purchaseInfo, folder);
+            overview.MarkUncommitedChange();
+        }
+
+        public void Stop()
+        {
+            cts.Cancel();
+            checkTask.Wait();
+        }
+    }
+}

Tools/AutoClient/Modes/Mode.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AutoClient.Modes
+{
+    public interface IMode
+    {
+        void Start(ICodexInstance instance, int index);
+        void Stop();
+    }
+}

Tools/AutoClient/Modes/PurchasingMode.cs

@@ -1,32 +1,23 @@
-using CodexOpenApi;
-using Logging;
-using Utils;
+using Logging;
 
-namespace AutoClient
+namespace AutoClient.Modes
 {
-    public class CodexUser
+    public class PurchasingMode : IMode
     {
+        private readonly List<AutomaticPurchaser> purchasers = new List<AutomaticPurchaser>();
         private readonly App app;
-        private readonly CodexApi codex;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly List<Purchaser> purchasers = new List<Purchaser>();
         private Task starterTask = Task.CompletedTask;
-        private readonly string nodeId = Guid.NewGuid().ToString();
 
-        public CodexUser(App app, CodexApi codex, HttpClient client, Address address)
+        public PurchasingMode(App app)
         {
             this.app = app;
-            this.codex = codex;
-            this.client = client;
-            this.address = address;
         }
 
-        public void Start(int index)
+        public void Start(ICodexInstance instance, int index)
         {
             for (var i = 0; i < app.Config.NumConcurrentPurchases; i++)
             {
-                purchasers.Add(new Purchaser(app, nodeId, new LogPrefixer(app.Log, $"({i}) "), client, address, codex));
+                purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance)));
             }
 
             var delayPerPurchaser =

Tools/AutoClient/Performance.cs

@@ -16,11 +16,13 @@ namespace AutoClient
             Log($"Download failed: {ex}");
         }
 
-        public void DownloadSuccessful(long size, TimeSpan time)
+        public void DownloadSuccessful(long? size, TimeSpan time)
         {
+            if (!size.HasValue) return;
+
             long milliseconds = Convert.ToInt64(time.TotalMilliseconds);
             if (milliseconds < 1) milliseconds = 1;
-            long bytesPerSecond = 1000 * (size / milliseconds);
+            long bytesPerSecond = 1000 * (size.Value / milliseconds);
             Log($"Download successful: {bytesPerSecond} bytes per second");
         }
 

Tools/AutoClient/Program.cs

@@ -1,11 +1,14 @@
 using ArgsUniform;
 using AutoClient;
+using AutoClient.Modes;
+using AutoClient.Modes.FolderStore;
 using CodexOpenApi;
 using Utils;
 
 public class Program
 {
     private readonly App app;
+    private readonly List<IMode> modes = new List<IMode>();
 
     public Program(Configuration config)
     {
@@ -31,36 +34,60 @@ public class Program
 
     public async Task Run()
     {
-        var codexUsers = await CreateUsers();
+        var codexInstances = await CreateCodexInstances();
 
         var i = 0;
-        foreach (var user in codexUsers)
+        foreach (var cdx in codexInstances)
         {
-            user.Start(i);
+            var mode = CreateMode();
+            modes.Add(mode);
+
+            mode.Start(cdx, i);
             i++;
         }
 
         app.Cts.Token.WaitHandle.WaitOne();
 
-        foreach (var user in codexUsers) user.Stop();
+        foreach (var mode in modes) mode.Stop();
+        modes.Clear();
 
         app.Log.Log("Done");
     }
 
-    private async Task<CodexUser[]> CreateUsers()
+    private IMode CreateMode()
+    {
+        if (!string.IsNullOrEmpty(app.Config.FolderToStore))
+        {
+            return CreateFolderStoreMode();
+        }
+
+        return new PurchasingMode(app);
+    }
+
+    private IMode CreateFolderStoreMode()
+    {
+        if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!");
+
+        return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo(
+            purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
+            purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120)
+        ));
+    }
+
+    private async Task<CodexInstance[]> CreateCodexInstances()
     {
         var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries);
-        var result = new List<CodexUser>();
+        var result = new List<CodexInstance>();
 
         foreach (var e in endpointStrs)
         {
-            result.Add(await CreateUser(e));
+            result.Add(await CreateCodexInstance(e));
         }
 
         return result.ToArray();
     }
 
-    private async Task<CodexUser> CreateUser(string endpoint)
+    private async Task<CodexInstance> CreateCodexInstance(string endpoint)
     {
         var splitIndex = endpoint.LastIndexOf(':');
         var host = endpoint.Substring(0, splitIndex);
@@ -72,6 +99,7 @@ public class Program
         );
 
         var client = new HttpClient();
+        client.Timeout = TimeSpan.FromMinutes(60.0);
         var codex = new CodexApi(client);
         codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
 
@@ -79,7 +107,7 @@ public class Program
         await CheckCodex(codex);
         app.Log.Log("OK");
 
-        return new CodexUser(
+        return new CodexInstance(
             app,
             codex,
             client,
@@ -105,4 +133,4 @@ public class Program
     {
         Console.WriteLine("Generates fake data and creates Codex storage contracts for it.");
     }
-}
+}

Tools/AutoClient/Purchaser.cs

@@ -1,258 +0,0 @@
-using CodexOpenApi;
-using CodexPlugin;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace AutoClient
-{
-    public class Purchaser
-    {
-        private readonly App app;
-        private readonly string nodeId;
-        private readonly ILog log;
-        private readonly HttpClient client;
-        private readonly Address address;
-        private readonly CodexApi codex;
-        private Task workerTask = Task.CompletedTask;
-
-        public Purchaser(App app, string nodeId, ILog log, HttpClient client, Address address, CodexApi codex)
-        {
-            this.app = app;
-            this.nodeId = nodeId;
-            this.log = log;
-            this.client = client;
-            this.address = address;
-            this.codex = codex;
-        }
-
-        public void Start()
-        {
-            workerTask = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            workerTask.Wait();
-        }
-
-        private async Task Worker()
-        {
-            log.Log("Worker started.");
-            while (!app.Cts.Token.IsCancellationRequested)
-            {
-                try
-                {
-                    var pid = await StartNewPurchase();
-                    await WaitTillFinished(pid);
-                    await DownloadForeignCid();
-                }
-                catch (Exception ex)
-                {
-                    log.Error("Worker failed with: " + ex);
-                    await Task.Delay(TimeSpan.FromHours(6));
-                }
-            }
-        }
-
-        private async Task DownloadForeignCid()
-        {
-            var cid = app.CidRepo.GetForeignCid(nodeId);
-            if (cid == null) return;
-            var size = app.CidRepo.GetSizeForCid(cid);
-            if (size == null) return;
-
-            try
-            {
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var filename = Guid.NewGuid().ToString().ToLowerInvariant();
-                {
-                    using var fileStream = File.OpenWrite(filename);
-                    var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
-                    fileResponse.Stream.CopyTo(fileStream);
-                }
-                var time = sw.Elapsed;
-                File.Delete(filename);
-                app.Performance.DownloadSuccessful(size.Value, time);
-            }
-            catch (Exception ex)
-            {
-                app.Performance.DownloadFailed(ex);
-            }
-        }
-
-        private async Task<string> StartNewPurchase()
-        {
-            var file = await CreateFile();
-            try
-            {
-                var cid = await UploadFile(file);
-                return await RequestStorage(cid);
-            }
-            finally
-            {
-                DeleteFile(file);
-            }
-        }
-
-        private async Task<string> CreateFile()
-        {
-            return await app.Generator.Generate();
-        }
-
-        private void DeleteFile(string file)
-        {
-            try
-            {
-                File.Delete(file);
-            }
-            catch (Exception exc)
-            {
-                app.Log.Error($"Failed to delete file '{file}': {exc}");
-            }
-        }
-
-        private async Task<ContentId> UploadFile(string filename)
-        {
-            using var fileStream = File.OpenRead(filename);
-            try
-            {
-                var info = new FileInfo(filename);
-                var sw = System.Diagnostics.Stopwatch.StartNew();
-                var cid = await UploadStream(fileStream);
-                var time = sw.Elapsed;
-                app.Performance.UploadSuccessful(info.Length, time);
-                app.CidRepo.Add(nodeId, cid.Id, info.Length);
-                return cid;
-            }
-            catch (Exception exc)
-            {
-                app.Performance.UploadFailed(exc);
-                throw;
-            }
-        }
-
-        private async Task<ContentId> UploadStream(FileStream fileStream)
-        {
-            log.Debug($"Uploading file...");
-            var response = await codex.UploadAsync(fileStream, app.Cts.Token);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block.");
-
-            log.Debug($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        private async Task<string> RequestStorage(ContentId cid)
-        {
-            log.Debug("Requesting storage for " + cid.Id);
-            var result = await codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation()
-            {
-                Collateral = app.Config.RequiredCollateral.ToString(),
-                Duration = (app.Config.ContractDurationMinutes * 60).ToString(),
-                Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(),
-                Nodes = app.Config.NumHosts,
-                Reward = app.Config.Price.ToString(),
-                ProofProbability = "15",
-                Tolerance = app.Config.HostTolerance
-            }, app.Cts.Token);
-
-            log.Debug("Purchase ID: " + result);
-
-            var encoded = await GetEncodedCid(result);
-            app.CidRepo.AddEncoded(cid.Id, encoded);
-
-            return result;
-        }
-
-        private async Task<string> GetEncodedCid(string pid)
-        {
-            try
-            {
-                var sp = (await GetStoragePurchase(pid))!;
-                return sp.Request.Content.Cid;
-            }
-            catch (Exception ex)
-            {
-                log.Error(ex.ToString());
-                throw;
-            }
-        }
-
-        private async Task<StoragePurchase?> GetStoragePurchase(string pid)
-        {
-            // openapi still don't match code.
-            var str = await client.GetStringAsync($"{address.Host}:{address.Port}/api/codex/v1/storage/purchases/{pid}");
-            if (string.IsNullOrEmpty(str)) return null;
-            return JsonConvert.DeserializeObject<StoragePurchase>(str);
-        }
-
-        private async Task WaitTillFinished(string pid)
-        {
-            try
-            {
-                var emptyResponseTolerance = 10;
-                while (!app.Cts.Token.IsCancellationRequested)
-                {
-                    var purchase = await GetStoragePurchase(pid);
-                    if (purchase == null)
-                    {
-                        await FixedShortDelay();
-                        emptyResponseTolerance--;
-                        if (emptyResponseTolerance == 0)
-                        {
-                            log.Log("Received 10 empty responses. Stop tracking this purchase.");
-                            await ExpiryTimeDelay();
-                            return;
-                        }
-                        continue;
-                    }
-                    var status = purchase.State.ToLowerInvariant();
-                    if (status.Contains("cancel"))
-                    {
-                        app.Performance.StorageContractCancelled();
-                        return;
-                    }
-                    if (status.Contains("error"))
-                    {
-                        app.Performance.StorageContractErrored(purchase.Error);
-                        return;
-                    }
-                    if (status.Contains("finished"))
-                    {
-                        app.Performance.StorageContractFinished();
-                        return;
-                    }
-                    if (status.Contains("started"))
-                    {
-                        app.Performance.StorageContractStarted();
-                        await FixedDurationDelay();
-                    }
-
-                    await FixedShortDelay();
-                }
-            }
-            catch (Exception ex)
-            {
-                log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time.");
-                await ExpiryTimeDelay();
-            }
-        }
-
-        private async Task FixedDurationDelay()
-        {
-            await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task ExpiryTimeDelay()
-        {
-            await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token);
-        }
-
-        private async Task FixedShortDelay()
-        {
-            await Task.Delay(15 * 1000, app.Cts.Token);
-        }
-    }
-}

Tools/BiblioTech/AdminChecker.cs

@@ -1,5 +1,7 @@
-using Discord;
+using BiblioTech.Options;
+using Discord;
 using Discord.WebSocket;
+using Org.BouncyCastle.Utilities;
 
 namespace BiblioTech
 {
@@ -29,7 +31,19 @@ namespace BiblioTech
 
         public async Task SendInAdminChannel(string msg)
         {
-            await adminChannel.SendMessageAsync(msg);
+            await SendInAdminChannel(msg.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        public async Task SendInAdminChannel(string[] lines)
+        {
+            var chunker = new LineChunker(lines);
+            var chunks = chunker.GetChunks();
+            if (!chunks.Any()) return;
+
+            foreach (var chunk in chunks)
+            {
+                await adminChannel.SendMessageAsync(string.Join(Environment.NewLine, chunk));
+            }
         }
 
         public void SetAdminChannel(ISocketMessageChannel adminChannel)

Tools/BiblioTech/CodexCidChecker.cs

@@ -1,5 +1,6 @@
 using CodexOpenApi;
 using IdentityModel.Client;
+using Logging;
 using Utils;
 
 namespace BiblioTech
@@ -8,11 +9,13 @@ namespace BiblioTech
     {
         private static readonly string nl = Environment.NewLine;
         private readonly Configuration config;
+        private readonly ILog log;
         private CodexApi? currentCodexNode;
 
-        public CodexCidChecker(Configuration config)
+        public CodexCidChecker(Configuration config, ILog log)
         {
             this.config = config;
+            this.log = log;
         }
 
         public async Task<CheckResponse> PerformCheck(string cid)
@@ -63,7 +66,7 @@ namespace BiblioTech
                 success: true,
                 title: $"Success: '{content.Cid}'",
                 error: "",
-                $"size: {content.Manifest.OriginalBytes} bytes",
+                $"size: {content.Manifest.DatasetSize} bytes",
                 $"blockSize: {content.Manifest.BlockSize} bytes",
                 $"protected: {content.Manifest.Protected}"
             );
@@ -150,6 +153,7 @@ namespace BiblioTech
             }
             catch (Exception e)
             {
+                log.Error(e.ToString());
                 return false;
             }
         }

Tools/BiblioTech/Program.cs

@@ -80,7 +80,7 @@ namespace BiblioTech
             client = new DiscordSocketClient();
             client.Log += ClientLog;
 
-            var checker = new CodexCidChecker(Config);
+            var checker = new CodexCidChecker(Config, Log);
             var notifyCommand = new NotifyCommand();
             var associateCommand = new UserAssociateCommand(notifyCommand);
             var sprCommand = new SprCommand();

Tools/CodexNetDeployer/Deployer.cs

@@ -161,7 +161,7 @@ namespace CodexNetDeployer
 
             Log("Starting metrics service...");
 
-            var runningContainer = ci.DeployMetricsCollector(startResults.Select(r => r.CodexNode).ToArray());
+            var runningContainer = ci.DeployMetricsCollector(scrapeInterval: TimeSpan.FromSeconds(10.0), startResults.Select(r => r.CodexNode).ToArray());
 
             Log("Metrics service started.");