logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-08-05 05:33:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Applies padding for folder file. Adds stats.

Browse Source
This commit is contained in:
ThatBen 2025-02-27 09:15:28 +01:00
parent 22e37c15e5
commit d701e2a73d
No known key found for this signature in database
GPG Key ID: 62C543548433D43E
3 changed files with 106 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
Tools/AutoClient/Modes/FolderStore/FileSaver.cs
View File

@ -8,13 +8,15 @@ namespace AutoClient.Modes.FolderStore
{
private readonly ILog log;
private readonly CodexWrapper instance;
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
public FileSaver(ILog log, CodexWrapper instance, string folderFile, FileStatus entry)
public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry)
{
this.log = log;
this.instance = instance;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
}
@ -87,11 +89,13 @@ namespace AutoClient.Modes.FolderStore
try
{
entry.BasicCid = instance.UploadFile(folderFile).Id;
stats.SuccessfulUploads++;
Log($"Successfully uploaded. BasicCid: '{entry.BasicCid}'");
}
catch (Exception exc)
{
entry.BasicCid = string.Empty;
stats.FailedUploads++;
log.Error("Failed to upload: " + exc);
HasFailed = true;
}
@ -103,15 +107,14 @@ namespace AutoClient.Modes.FolderStore
try
{
var request = instance.RequestStorage(new ContentId(entry.BasicCid));
entry.EncodedCid = request.Purchase.ContentId.Id;
entry.PurchaseId = request.PurchaseId;
var request = CreateNewStorageRequest();
request.WaitForStorageContractSubmitted();
request.WaitForStorageContractStarted();
WaitForSubmitted(request);
WaitForStarted(request);
entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration;
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)} ");
stats.StorageRequestStats.SuccessfullyStarted++;
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}");
}
catch (Exception exc)
{
@ -122,6 +125,48 @@ namespace AutoClient.Modes.FolderStore
}
}
private IStoragePurchaseContract CreateNewStorageRequest()
{
try
{
var request = instance.RequestStorage(new ContentId(entry.BasicCid));
entry.EncodedCid = request.Purchase.ContentId.Id;
entry.PurchaseId = request.PurchaseId;
return request;
}
catch
{
stats.StorageRequestStats.FailedToCreate++;
throw;
}
}
private void WaitForSubmitted(IStoragePurchaseContract request)
{
try
{
request.WaitForStorageContractSubmitted();
}
catch
{
stats.StorageRequestStats.FailedToSubmit++;
throw;
}
}
private void WaitForStarted(IStoragePurchaseContract request)
{
try
{
request.WaitForStorageContractStarted();
}
catch
{
stats.StorageRequestStats.FailedToStart++;
throw;
}
}
private void Log(string msg)
{
log.Log(msg);

39
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs
View File

@ -1,5 +1,4 @@
using CodexClient;
using Logging;
using Logging;
namespace AutoClient.Modes.FolderStore
{
@ -50,6 +49,7 @@ namespace AutoClient.Modes.FolderStore
SaveFolderSaverJsonFile();
}
statusFile.Save(status);
Thread.Sleep(2000);
}
}
@ -67,7 +67,6 @@ namespace AutoClient.Modes.FolderStore
status.Files.Add(entry);
}
ProcessFileEntry(folderFile, entry);
statusFile.Save(status);
}
private void ProcessFileEntry(string folderFile, FileStatus entry)
@ -84,16 +83,48 @@ namespace AutoClient.Modes.FolderStore
Filename = FolderSaverFilename
};
var folderFile = Path.Combine(app.Config.FolderToStore, FolderSaverFilename);
ApplyPadding(folderFile);
var fileSaver = CreateFileSaver(folderFile, entry);
fileSaver.Process();
if (fileSaver.HasFailed) failureCount++;
}
private const int MinCodexStorageFilesize = 262144;
private readonly Random random = new Random();
private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " +
$"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " +
$"file is larger than the minimal size. The following is pseudo-random: ";
private void ApplyPadding(string folderFile)
{
var info = new FileInfo(folderFile);
var min = MinCodexStorageFilesize * 2;
if (info.Length < min)
{
var required = min - info.Length;
status.Padding = paddingMessage + GenerateRandomString(required);
}
}
private string GenerateRandomString(long required)
{
var result = "";
while (result.Length < required)
{
var diff = required - result.Length;
var bytes = new byte[diff];
random.NextBytes(bytes);
result += string.Join("", bytes.Select(b => b.ToString()));
}
return result;
}
private FileSaver CreateFileSaver(string folderFile, FileStatus entry)
{
var fixedLength = entry.Filename.PadRight(35);
var prefix = $"[{fixedLength}] ";
return new FileSaver(new LogPrefixer(app.Log, prefix), instance, folderFile, entry);
return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry);
}
}
}

19
Tools/AutoClient/Modes/FolderStore/FolderStatus.cs
View File

@ -4,6 +4,8 @@
public class FolderStatus
{
public List<FileStatus> Files { get; set; } = new List<FileStatus>();
public Stats Stats { get; set; } = new Stats();
public string Padding { get; set; } = string.Empty;
}
[Serializable]
@ -15,4 +17,21 @@
public string PurchaseId { get; set; } = string.Empty;
public DateTime PurchaseFinishedUtc { get; set; } = DateTime.MinValue;
}
[Serializable]
public class Stats
{
public int SuccessfulUploads { get; set; }
public int FailedUploads { get; set; }
public StorageRequestStats StorageRequestStats { get; set; } = new StorageRequestStats();
}
[Serializable]
public class StorageRequestStats
{
public int FailedToCreate { get; set; }
public int FailedToSubmit { get; set; }
public int FailedToStart { get; set; }
public int SuccessfullyStarted { get; set; }
}
}