diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 13aea6fd..65996333 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -59,6 +59,9 @@ namespace AutoClient "/root/codex-testnet-starter/scripts/eth_7.address" + ";" + "/root/codex-testnet-starter/scripts/eth_8.address"; + [Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")] + public int SlowModeDelayMinutes { get; set; } = 60 * 1; + public string LogPath { get diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index 027065e4..448755f0 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore public interface IFileSaverEventHandler { void SaveChanges(); + } + + public interface IFileSaverResultHandler + { + void OnSuccess(); void OnFailure(); } @@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; - public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.loadBalancer = loadBalancer; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; } public void Process() @@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore loadBalancer.DispatchOnCodex(instance => { entry.CodexNodeId = instance.Node.GetName(); - handler.SaveChanges(); + saveHandler.SaveChanges(); - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }); } @@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore { loadBalancer.DispatchOnSpecificCodex(instance => { - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }, entry.CodexNodeId); } @@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; private readonly QuotaCheck quotaCheck; - public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.instance = instance; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; quotaCheck = new QuotaCheck(log, folderFile, instance); } @@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore Thread.Sleep(TimeSpan.FromMinutes(1.0)); } Log("Could not upload: Insufficient local storage quota."); - handler.OnFailure(); + resultHandler.OnFailure(); return false; } @@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore entry.BasicCid = string.Empty; stats.FailedUploads++; log.Error("Failed to upload: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } - handler.SaveChanges(); + saveHandler.SaveChanges(); } private void CreateNewPurchase() @@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore WaitForStarted(request); stats.StorageRequestStats.SuccessfullyStarted++; - handler.SaveChanges(); + saveHandler.SaveChanges(); Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); + resultHandler.OnSuccess(); } catch (Exception exc) { entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); log.Error("Failed to start new purchase: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } } @@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore throw new Exception("CID received from storage request was not protected."); } - handler.SaveChanges(); + saveHandler.SaveChanges(); Log("Saved new purchaseId: " + entry.PurchaseId); return request; } @@ -289,7 +299,7 @@ namespace AutoClient.Modes.FolderStore Log("Request failed to start. State: " + update.State); entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); return; } } @@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore } catch (Exception exc) { - handler.OnFailure(); + resultHandler.OnFailure(); Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}"); throw; } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index 4d090e6d..d951aaee 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -11,14 +11,16 @@ namespace AutoClient.Modes.FolderStore private readonly JsonFile statusFile; private readonly FolderStatus status; private readonly BalanceChecker balanceChecker; + private readonly SlowModeHandler slowModeHandler; private int changeCounter = 0; - private int failureCount = 0; + private int saveFolderJsonCounter = 0; public FolderSaver(App app, LoadBalancer loadBalancer) { this.app = app; this.loadBalancer = loadBalancer; balanceChecker = new BalanceChecker(app); + slowModeHandler = new SlowModeHandler(app); statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); status = statusFile.Load(); @@ -26,10 +28,11 @@ namespace AutoClient.Modes.FolderStore public void Run() { + saveFolderJsonCounter = 0; + var folderFiles = Directory.GetFiles(app.Config.FolderToStore); if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); - var saveFolderJsonCounter = 0; balanceChecker.Check(); foreach (var folderFile in folderFiles) { @@ -41,35 +44,30 @@ namespace AutoClient.Modes.FolderStore SaveFile(folderFile); } - if (failureCount > 3) - { - app.Log.Error("Failure count reached threshold. Stopping..."); - app.Cts.Cancel(); - return; - } - - if (changeCounter > 1) - { - changeCounter = 0; - saveFolderJsonCounter++; - if (saveFolderJsonCounter > 5) - { - saveFolderJsonCounter = 0; - if (failureCount > 0) - { - app.Log.Log($"Failure count is reset. (Was: {failureCount})"); - failureCount = 0; - } - balanceChecker.Check(); - SaveFolderSaverJsonFile(); - } - } + slowModeHandler.Check(); + + CheckAndSaveChanges(); statusFile.Save(status); Thread.Sleep(100); } } + private void CheckAndSaveChanges() + { + if (changeCounter > 1) + { + changeCounter = 0; + saveFolderJsonCounter++; + if (saveFolderJsonCounter > 5) + { + saveFolderJsonCounter = 0; + balanceChecker.Check(); + SaveFolderSaverJsonFile(); + } + } + } + private void SaveFile(string folderFile) { var localFilename = Path.GetFileName(folderFile); @@ -114,7 +112,6 @@ namespace AutoClient.Modes.FolderStore } private const int MinCodexStorageFilesize = 262144; - private readonly Random random = new Random(); private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " + $"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " + $"file is larger than the minimal size. The following is pseudo-random: "; @@ -135,7 +132,7 @@ namespace AutoClient.Modes.FolderStore { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this); + return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler); } public void SaveChanges() @@ -143,10 +140,5 @@ namespace AutoClient.Modes.FolderStore statusFile.Save(status); changeCounter++; } - - public void OnFailure() - { - failureCount++; - } } } diff --git a/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs new file mode 100644 index 00000000..37d3f7ee --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs @@ -0,0 +1,54 @@ +namespace AutoClient.Modes.FolderStore +{ + public class SlowModeHandler : IFileSaverResultHandler + { + private readonly App app; + private int failureCount = 0; + private bool slowMode = false; + private int recoveryCount = 0; + + public SlowModeHandler(App app) + { + this.app = app; + } + + public void OnSuccess() + { + failureCount = 0; + if (slowMode) + { + recoveryCount++; + if (recoveryCount > 3) + { + Log("Recovery limit reached. Exiting slow mode."); + slowMode = false; + failureCount = 0; + } + } + } + + public void OnFailure() + { + failureCount++; + if (failureCount > 3 && !slowMode) + { + Log("Failure limit reached. Entering slow mode."); + slowMode = true; + recoveryCount = 0; + } + } + + public void Check() + { + if (slowMode) + { + Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes)); + } + } + + private void Log(string msg) + { + app.Log.Log(msg); + } + } +}