diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index dad87d89..179dbee7 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -8,13 +8,15 @@ namespace AutoClient.Modes.FolderStore { private readonly ILog log; private readonly CodexWrapper instance; + private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - public FileSaver(ILog log, CodexWrapper instance, string folderFile, FileStatus entry) + public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry) { this.log = log; this.instance = instance; + this.stats = stats; this.folderFile = folderFile; this.entry = entry; } @@ -87,11 +89,13 @@ namespace AutoClient.Modes.FolderStore try { entry.BasicCid = instance.UploadFile(folderFile).Id; + stats.SuccessfulUploads++; Log($"Successfully uploaded. BasicCid: '{entry.BasicCid}'"); } catch (Exception exc) { entry.BasicCid = string.Empty; + stats.FailedUploads++; log.Error("Failed to upload: " + exc); HasFailed = true; } @@ -103,15 +107,14 @@ namespace AutoClient.Modes.FolderStore try { - var request = instance.RequestStorage(new ContentId(entry.BasicCid)); - entry.EncodedCid = request.Purchase.ContentId.Id; - entry.PurchaseId = request.PurchaseId; + var request = CreateNewStorageRequest(); - request.WaitForStorageContractSubmitted(); - request.WaitForStorageContractStarted(); + WaitForSubmitted(request); + WaitForStarted(request); entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration; - Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)} "); + stats.StorageRequestStats.SuccessfullyStarted++; + Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); } catch (Exception exc) { @@ -122,6 +125,48 @@ namespace AutoClient.Modes.FolderStore } } + private IStoragePurchaseContract CreateNewStorageRequest() + { + try + { + var request = instance.RequestStorage(new ContentId(entry.BasicCid)); + entry.EncodedCid = request.Purchase.ContentId.Id; + entry.PurchaseId = request.PurchaseId; + return request; + } + catch + { + stats.StorageRequestStats.FailedToCreate++; + throw; + } + } + + private void WaitForSubmitted(IStoragePurchaseContract request) + { + try + { + request.WaitForStorageContractSubmitted(); + } + catch + { + stats.StorageRequestStats.FailedToSubmit++; + throw; + } + } + + private void WaitForStarted(IStoragePurchaseContract request) + { + try + { + request.WaitForStorageContractStarted(); + } + catch + { + stats.StorageRequestStats.FailedToStart++; + throw; + } + } + private void Log(string msg) { log.Log(msg); diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index b03da4d7..40ca5773 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -1,5 +1,4 @@ -using CodexClient; -using Logging; +using Logging; namespace AutoClient.Modes.FolderStore { @@ -50,6 +49,7 @@ namespace AutoClient.Modes.FolderStore SaveFolderSaverJsonFile(); } + statusFile.Save(status); Thread.Sleep(2000); } } @@ -67,7 +67,6 @@ namespace AutoClient.Modes.FolderStore status.Files.Add(entry); } ProcessFileEntry(folderFile, entry); - statusFile.Save(status); } private void ProcessFileEntry(string folderFile, FileStatus entry) @@ -84,16 +83,48 @@ namespace AutoClient.Modes.FolderStore Filename = FolderSaverFilename }; var folderFile = Path.Combine(app.Config.FolderToStore, FolderSaverFilename); + ApplyPadding(folderFile); var fileSaver = CreateFileSaver(folderFile, entry); fileSaver.Process(); if (fileSaver.HasFailed) failureCount++; } + private const int MinCodexStorageFilesize = 262144; + private readonly Random random = new Random(); + private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " + + $"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " + + $"file is larger than the minimal size. The following is pseudo-random: "; + + private void ApplyPadding(string folderFile) + { + var info = new FileInfo(folderFile); + var min = MinCodexStorageFilesize * 2; + if (info.Length < min) + { + var required = min - info.Length; + status.Padding = paddingMessage + GenerateRandomString(required); + } + } + + private string GenerateRandomString(long required) + { + var result = ""; + while (result.Length < required) + { + var diff = required - result.Length; + var bytes = new byte[diff]; + random.NextBytes(bytes); + result += string.Join("", bytes.Select(b => b.ToString())); + } + + return result; + } + private FileSaver CreateFileSaver(string folderFile, FileStatus entry) { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), instance, folderFile, entry); + return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry); } } } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs index 9b68b7ce..e9b7ec14 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs @@ -4,6 +4,8 @@ public class FolderStatus { public List Files { get; set; } = new List(); + public Stats Stats { get; set; } = new Stats(); + public string Padding { get; set; } = string.Empty; } [Serializable] @@ -15,4 +17,21 @@ public string PurchaseId { get; set; } = string.Empty; public DateTime PurchaseFinishedUtc { get; set; } = DateTime.MinValue; } + + [Serializable] + public class Stats + { + public int SuccessfulUploads { get; set; } + public int FailedUploads { get; set; } + public StorageRequestStats StorageRequestStats { get; set; } = new StorageRequestStats(); + } + + [Serializable] + public class StorageRequestStats + { + public int FailedToCreate { get; set; } + public int FailedToSubmit { get; set; } + public int FailedToStart { get; set; } + public int SuccessfullyStarted { get; set; } + } }