logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-08-09 15:43:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

immediate saves, resumable waiting

Browse Source
This commit is contained in:
ThatBen 2025-02-28 16:19:26 +01:00
parent eaa69bcc95
commit 7e37299deb
No known key found for this signature in database
GPG Key ID: 62C543548433D43E
2 changed files with 69 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
Tools/AutoClient/Modes/FolderStore/FileSaver.cs
View File

@ -11,27 +11,26 @@ namespace AutoClient.Modes.FolderStore
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
private readonly Action saveChanges;
private readonly QuotaCheck quotaCheck;
public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry)
public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, Action saveChanges)
{
this.log = log;
this.instance = instance;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
this.saveChanges = saveChanges;
quotaCheck = new QuotaCheck(log, folderFile, instance);
}
public bool HasFailed { get; private set; }
public bool Changes { get; private set; }
public void Process()
{
HasFailed = false;
Changes = false;
if (HasRecentPurchase(entry))
if (HasRecentPurchase())
{
Log($"Purchase running: '{entry.PurchaseId}'");
return;
@ -76,11 +75,15 @@ namespace AutoClient.Modes.FolderStore
return false;
}
private bool HasRecentPurchase(FileStatus entry)
private bool HasRecentPurchase()
{
if (string.IsNullOrEmpty(entry.PurchaseId)) return false;
var purchase = GetPurchase(entry.PurchaseId);
if (purchase == null) return false;
if (purchase.IsSubmitted)
{
WaitForSubmittedToStarted(purchase);
}
if (!purchase.IsStarted) return false;
// Purchase is started. But, if it finishes soon, we will treat it as already finished.
@ -121,7 +124,6 @@ namespace AutoClient.Modes.FolderStore
private void UploadFile()
{
Log("Uploading file...");
Changes = true;
try
{
entry.BasicCid = instance.UploadFile(folderFile).Id;
@ -135,6 +137,7 @@ namespace AutoClient.Modes.FolderStore
log.Error("Failed to upload: " + exc);
HasFailed = true;
}
saveChanges();
}
private void CreateNewPurchase()
@ -142,22 +145,23 @@ namespace AutoClient.Modes.FolderStore
if (string.IsNullOrEmpty(entry.BasicCid)) return;
Log("Creating new purchase...");
Changes = true;
try
{
var request = CreateNewStorageRequest();
entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration;
stats.StorageRequestStats.SuccessfullyStarted++;
saveChanges();
WaitForSubmitted(request);
WaitForStarted(request);
entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration;
stats.StorageRequestStats.SuccessfullyStarted++;
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}");
}
catch (Exception exc)
{
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
saveChanges();
log.Error("Failed to start new purchase: " + exc);
HasFailed = true;
}
@ -179,6 +183,45 @@ namespace AutoClient.Modes.FolderStore
}
}
private void WaitForSubmittedToStarted(StoragePurchase purchase)
{
try
{
var expirySeconds = Convert.ToInt64(purchase.Request.Expiry);
var expiry = TimeSpan.FromSeconds(expirySeconds);
Log($"Request was submitted but not started yet. Waiting {Time.FormatDuration(expiry)} to start or expire...");
var limit = DateTime.UtcNow + expiry;
while (DateTime.UtcNow < limit)
{
Thread.Sleep(TimeSpan.FromSeconds(30));
var update = GetPurchase(purchase.Request.Id);
if (update != null)
{
if (update.IsStarted)
{
Log("Request successfully started.");
return;
}
else if (!update.IsSubmitted)
{
Log("Request failed to start. State: " + update.State);
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
saveChanges();
return;
}
}
}
}
catch (Exception exc)
{
HasFailed = true;
Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}");
throw;
}
}
private void WaitForSubmitted(IStoragePurchaseContract request)
{
try

28
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs
View File

@ -9,6 +9,7 @@ namespace AutoClient.Modes.FolderStore
private readonly CodexWrapper instance;
private readonly JsonFile<FolderStatus> statusFile;
private readonly FolderStatus status;
private int changeCounter = 0;
private int failureCount = 0;
public FolderSaver(App app, CodexWrapper instance)
@ -25,17 +26,14 @@ namespace AutoClient.Modes.FolderStore
var folderFiles = Directory.GetFiles(app.Config.FolderToStore);
if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore);
var counter = 0;
changeCounter = 0;
foreach (var folderFile in folderFiles)
{
if (cts.IsCancellationRequested) return;
if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename))
{
if (SaveFile(folderFile))
{
counter++;
}
SaveFile(folderFile);
}
if (failureCount > 9)
@ -45,9 +43,9 @@ namespace AutoClient.Modes.FolderStore
return;
}
if (counter > 5)
if (changeCounter > 10)
{
counter = 0;
changeCounter = 0;
SaveFolderSaverJsonFile();
}
@ -56,7 +54,7 @@ namespace AutoClient.Modes.FolderStore
}
}
private bool SaveFile(string folderFile)
private void SaveFile(string folderFile)
{
var localFilename = Path.GetFileName(folderFile);
var entry = status.Files.SingleOrDefault(f => f.Filename == localFilename);
@ -68,19 +66,19 @@ namespace AutoClient.Modes.FolderStore
};
status.Files.Add(entry);
}
return ProcessFileEntry(folderFile, entry);
ProcessFileEntry(folderFile, entry);
}
private bool ProcessFileEntry(string folderFile, FileStatus entry)
private void ProcessFileEntry(string folderFile, FileStatus entry)
{
var fileSaver = CreateFileSaver(folderFile, entry);
fileSaver.Process();
if (fileSaver.HasFailed) failureCount++;
return fileSaver.Changes;
}
private void SaveFolderSaverJsonFile()
{
app.Log.Log($"Saving {FolderSaverFilename}...");
var entry = new FileStatus
{
Filename = FolderSaverFilename
@ -90,6 +88,8 @@ namespace AutoClient.Modes.FolderStore
var fileSaver = CreateFileSaver(folderFile, entry);
fileSaver.Process();
if (fileSaver.HasFailed) failureCount++;
app.Log.Log($"!!! {FolderSaverFilename} saved to CID '{entry.EncodedCid}' !!!");
}
private const int MinCodexStorageFilesize = 262144;
@ -127,7 +127,11 @@ namespace AutoClient.Modes.FolderStore
{
var fixedLength = entry.Filename.PadRight(35);
var prefix = $"[{fixedLength}] ";
return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry);
return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry, saveChanges: () =>
{
statusFile.Save(status);
changeCounter++;
});
}
}
}