logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-02-11 16:43:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement slow mode and recovery

Browse Source
This commit is contained in:
ThatBen 2025-05-02 08:21:56 +02:00
parent 17ffb41e15
commit c68d4bb13f
No known key found for this signature in database
GPG Key ID: E020A7DDCD52E1AB
4 changed files with 109 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Tools/AutoClient/Configuration.cs
View File

@ -59,6 +59,9 @@ namespace AutoClient
"/root/codex-testnet-starter/scripts/eth_7.address" + ";" +
"/root/codex-testnet-starter/scripts/eth_8.address";
[Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")]
public int SlowModeDelayMinutes { get; set; } = 60 * 1;
public string LogPath
{
get

46
Tools/AutoClient/Modes/FolderStore/FileSaver.cs
View File

@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore
public interface IFileSaverEventHandler
{
void SaveChanges();
}
public interface IFileSaverResultHandler
{
void OnSuccess();
void OnFailure();
}
@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
private readonly IFileSaverEventHandler handler;
private readonly IFileSaverEventHandler saveHandler;
private readonly IFileSaverResultHandler resultHandler;
public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
{
this.log = log;
this.loadBalancer = loadBalancer;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
this.handler = handler;
this.saveHandler = saveHandler;
this.resultHandler = resultHandler;
}
public void Process()
@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore
loadBalancer.DispatchOnCodex(instance =>
{
entry.CodexNodeId = instance.Node.GetName();
handler.SaveChanges();
saveHandler.SaveChanges();
var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
run.Process();
});
}
@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore
{
loadBalancer.DispatchOnSpecificCodex(instance =>
{
var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
run.Process();
}, entry.CodexNodeId);
}
@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
private readonly IFileSaverEventHandler handler;
private readonly IFileSaverEventHandler saveHandler;
private readonly IFileSaverResultHandler resultHandler;
private readonly QuotaCheck quotaCheck;
public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
{
this.log = log;
this.instance = instance;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
this.handler = handler;
this.saveHandler = saveHandler;
this.resultHandler = resultHandler;
quotaCheck = new QuotaCheck(log, folderFile, instance);
}
@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore
Thread.Sleep(TimeSpan.FromMinutes(1.0));
}
Log("Could not upload: Insufficient local storage quota.");
handler.OnFailure();
resultHandler.OnFailure();
return false;
}
@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore
entry.BasicCid = string.Empty;
stats.FailedUploads++;
log.Error("Failed to upload: " + exc);
handler.OnFailure();
resultHandler.OnFailure();
}
handler.SaveChanges();
saveHandler.SaveChanges();
}
private void CreateNewPurchase()
@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore
WaitForStarted(request);
stats.StorageRequestStats.SuccessfullyStarted++;
handler.SaveChanges();
saveHandler.SaveChanges();
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}");
resultHandler.OnSuccess();
}
catch (Exception exc)
{
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
handler.SaveChanges();
saveHandler.SaveChanges();
log.Error("Failed to start new purchase: " + exc);
handler.OnFailure();
resultHandler.OnFailure();
}
}
@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore
throw new Exception("CID received from storage request was not protected.");
}
handler.SaveChanges();
saveHandler.SaveChanges();
Log("Saved new purchaseId: " + entry.PurchaseId);
return request;
}
@ -289,7 +299,7 @@ namespace AutoClient.Modes.FolderStore
Log("Request failed to start. State: " + update.State);
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
handler.SaveChanges();
saveHandler.SaveChanges();
return;
}
}
@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore
}
catch (Exception exc)
{
handler.OnFailure();
resultHandler.OnFailure();
Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}");
throw;
}

56
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs
View File

@ -11,14 +11,16 @@ namespace AutoClient.Modes.FolderStore
private readonly JsonFile<FolderStatus> statusFile;
private readonly FolderStatus status;
private readonly BalanceChecker balanceChecker;
private readonly SlowModeHandler slowModeHandler;
private int changeCounter = 0;
private int failureCount = 0;
private int saveFolderJsonCounter = 0;
public FolderSaver(App app, LoadBalancer loadBalancer)
{
this.app = app;
this.loadBalancer = loadBalancer;
balanceChecker = new BalanceChecker(app);
slowModeHandler = new SlowModeHandler(app);
statusFile = new JsonFile<FolderStatus>(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename));
status = statusFile.Load();
@ -26,10 +28,11 @@ namespace AutoClient.Modes.FolderStore
public void Run()
{
saveFolderJsonCounter = 0;
var folderFiles = Directory.GetFiles(app.Config.FolderToStore);
if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore);
var saveFolderJsonCounter = 0;
balanceChecker.Check();
foreach (var folderFile in folderFiles)
{
@ -41,35 +44,30 @@ namespace AutoClient.Modes.FolderStore
SaveFile(folderFile);
}
if (failureCount > 3)
{
app.Log.Error("Failure count reached threshold. Stopping...");
app.Cts.Cancel();
return;
}
if (changeCounter > 1)
{
changeCounter = 0;
saveFolderJsonCounter++;
if (saveFolderJsonCounter > 5)
{
saveFolderJsonCounter = 0;
if (failureCount > 0)
{
app.Log.Log($"Failure count is reset. (Was: {failureCount})");
failureCount = 0;
}
balanceChecker.Check();
SaveFolderSaverJsonFile();
}
}
slowModeHandler.Check();
CheckAndSaveChanges();
statusFile.Save(status);
Thread.Sleep(100);
}
}
private void CheckAndSaveChanges()
{
if (changeCounter > 1)
{
changeCounter = 0;
saveFolderJsonCounter++;
if (saveFolderJsonCounter > 5)
{
saveFolderJsonCounter = 0;
balanceChecker.Check();
SaveFolderSaverJsonFile();
}
}
}
private void SaveFile(string folderFile)
{
var localFilename = Path.GetFileName(folderFile);
@ -114,7 +112,6 @@ namespace AutoClient.Modes.FolderStore
}
private const int MinCodexStorageFilesize = 262144;
private readonly Random random = new Random();
private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " +
$"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " +
$"file is larger than the minimal size. The following is pseudo-random: ";
@ -135,7 +132,7 @@ namespace AutoClient.Modes.FolderStore
{
var fixedLength = entry.Filename.PadRight(35);
var prefix = $"[{fixedLength}] ";
return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this);
return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler);
}
public void SaveChanges()
@ -143,10 +140,5 @@ namespace AutoClient.Modes.FolderStore
statusFile.Save(status);
changeCounter++;
}
public void OnFailure()
{
failureCount++;
}
}
}

54
Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs Normal file
View File

@ -0,0 +1,54 @@
namespace AutoClient.Modes.FolderStore
{
public class SlowModeHandler : IFileSaverResultHandler
{
private readonly App app;
private int failureCount = 0;
private bool slowMode = false;
private int recoveryCount = 0;
public SlowModeHandler(App app)
{
this.app = app;
}
public void OnSuccess()
{
failureCount = 0;
if (slowMode)
{
recoveryCount++;
if (recoveryCount > 3)
{
Log("Recovery limit reached. Exiting slow mode.");
slowMode = false;
failureCount = 0;
}
}
}
public void OnFailure()
{
failureCount++;
if (failureCount > 3 && !slowMode)
{
Log("Failure limit reached. Entering slow mode.");
slowMode = true;
recoveryCount = 0;
}
}
public void Check()
{
if (slowMode)
{
Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes));
}
}
private void Log(string msg)
{
app.Log.Log(msg);
}
}
}