From 17ffb41e150b5b133697cc99707b80626ea2a562 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 2 May 2025 08:01:44 +0200 Subject: [PATCH 1/4] Makes proof period report interval configurable default to 24 hours --- Tools/TestNetRewarder/Configuration.cs | 3 +++ Tools/TestNetRewarder/Processor.cs | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Tools/TestNetRewarder/Configuration.cs b/Tools/TestNetRewarder/Configuration.cs index 5dbeac30..b3aa84cb 100644 --- a/Tools/TestNetRewarder/Configuration.cs +++ b/Tools/TestNetRewarder/Configuration.cs @@ -33,6 +33,9 @@ namespace TestNetRewarder [Uniform("proof-submitted-events", "pse", "PROOFSUBMITTEDEVENTS", false, "When greater than zero, chain event summary will include proof-submitted events.")] public int ShowProofSubmittedEvents { get; set; } = 0; // Defaulted to zero, aprox 7 to 10 such events every 2 minutes in testnet (from autoclient alone!) + [Uniform("proof-period-report-hours", "pprh", "PROOFPERIODREPORTHOURS", false, "Frequency in hours with which proof period reports are created.")] + public int ProofReportHours { get; set; } = 24; + public string LogPath { get diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index eed26962..649ebf47 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -22,6 +22,8 @@ namespace TestNetRewarder this.log = log; lastPeriodUpdateUtc = DateTime.UtcNow; + if (config.ProofReportHours < 1) throw new Exception("ProofReportHours must be one or greater"); + builder = new RequestBuilder(); eventsFormatter = new EventsFormatter(config); @@ -79,7 +81,7 @@ namespace TestNetRewarder private void ProcessPeriodUpdate() { if (config.ShowProofPeriodReports < 1) return; - if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(1.0))) return; + if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(config.ProofReportHours))) return; lastPeriodUpdateUtc = DateTime.UtcNow; eventsFormatter.ProcessPeriodReports(chainState.PeriodMonitor.GetAndClearReports()); From c68d4bb13f073c4c632f2f83ebf2a389934eb497 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 2 May 2025 08:21:56 +0200 Subject: [PATCH 2/4] Implement slow mode and recovery --- Tools/AutoClient/Configuration.cs | 3 + .../AutoClient/Modes/FolderStore/FileSaver.cs | 46 +++++++++------ .../Modes/FolderStore/FolderSaver.cs | 56 ++++++++----------- .../Modes/FolderStore/SlowModeHandler.cs | 54 ++++++++++++++++++ 4 files changed, 109 insertions(+), 50 deletions(-) create mode 100644 Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 13aea6fd..65996333 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -59,6 +59,9 @@ namespace AutoClient "/root/codex-testnet-starter/scripts/eth_7.address" + ";" + "/root/codex-testnet-starter/scripts/eth_8.address"; + [Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")] + public int SlowModeDelayMinutes { get; set; } = 60 * 1; + public string LogPath { get diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index 027065e4..448755f0 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore public interface IFileSaverEventHandler { void SaveChanges(); + } + + public interface IFileSaverResultHandler + { + void OnSuccess(); void OnFailure(); } @@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; - public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.loadBalancer = loadBalancer; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; } public void Process() @@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore loadBalancer.DispatchOnCodex(instance => { entry.CodexNodeId = instance.Node.GetName(); - handler.SaveChanges(); + saveHandler.SaveChanges(); - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }); } @@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore { loadBalancer.DispatchOnSpecificCodex(instance => { - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }, entry.CodexNodeId); } @@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; private readonly QuotaCheck quotaCheck; - public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.instance = instance; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; quotaCheck = new QuotaCheck(log, folderFile, instance); } @@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore Thread.Sleep(TimeSpan.FromMinutes(1.0)); } Log("Could not upload: Insufficient local storage quota."); - handler.OnFailure(); + resultHandler.OnFailure(); return false; } @@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore entry.BasicCid = string.Empty; stats.FailedUploads++; log.Error("Failed to upload: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } - handler.SaveChanges(); + saveHandler.SaveChanges(); } private void CreateNewPurchase() @@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore WaitForStarted(request); stats.StorageRequestStats.SuccessfullyStarted++; - handler.SaveChanges(); + saveHandler.SaveChanges(); Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); + resultHandler.OnSuccess(); } catch (Exception exc) { entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); log.Error("Failed to start new purchase: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } } @@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore throw new Exception("CID received from storage request was not protected."); } - handler.SaveChanges(); + saveHandler.SaveChanges(); Log("Saved new purchaseId: " + entry.PurchaseId); return request; } @@ -289,7 +299,7 @@ namespace AutoClient.Modes.FolderStore Log("Request failed to start. State: " + update.State); entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); return; } } @@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore } catch (Exception exc) { - handler.OnFailure(); + resultHandler.OnFailure(); Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}"); throw; } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index 4d090e6d..d951aaee 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -11,14 +11,16 @@ namespace AutoClient.Modes.FolderStore private readonly JsonFile statusFile; private readonly FolderStatus status; private readonly BalanceChecker balanceChecker; + private readonly SlowModeHandler slowModeHandler; private int changeCounter = 0; - private int failureCount = 0; + private int saveFolderJsonCounter = 0; public FolderSaver(App app, LoadBalancer loadBalancer) { this.app = app; this.loadBalancer = loadBalancer; balanceChecker = new BalanceChecker(app); + slowModeHandler = new SlowModeHandler(app); statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); status = statusFile.Load(); @@ -26,10 +28,11 @@ namespace AutoClient.Modes.FolderStore public void Run() { + saveFolderJsonCounter = 0; + var folderFiles = Directory.GetFiles(app.Config.FolderToStore); if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); - var saveFolderJsonCounter = 0; balanceChecker.Check(); foreach (var folderFile in folderFiles) { @@ -41,35 +44,30 @@ namespace AutoClient.Modes.FolderStore SaveFile(folderFile); } - if (failureCount > 3) - { - app.Log.Error("Failure count reached threshold. Stopping..."); - app.Cts.Cancel(); - return; - } - - if (changeCounter > 1) - { - changeCounter = 0; - saveFolderJsonCounter++; - if (saveFolderJsonCounter > 5) - { - saveFolderJsonCounter = 0; - if (failureCount > 0) - { - app.Log.Log($"Failure count is reset. (Was: {failureCount})"); - failureCount = 0; - } - balanceChecker.Check(); - SaveFolderSaverJsonFile(); - } - } + slowModeHandler.Check(); + + CheckAndSaveChanges(); statusFile.Save(status); Thread.Sleep(100); } } + private void CheckAndSaveChanges() + { + if (changeCounter > 1) + { + changeCounter = 0; + saveFolderJsonCounter++; + if (saveFolderJsonCounter > 5) + { + saveFolderJsonCounter = 0; + balanceChecker.Check(); + SaveFolderSaverJsonFile(); + } + } + } + private void SaveFile(string folderFile) { var localFilename = Path.GetFileName(folderFile); @@ -114,7 +112,6 @@ namespace AutoClient.Modes.FolderStore } private const int MinCodexStorageFilesize = 262144; - private readonly Random random = new Random(); private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " + $"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " + $"file is larger than the minimal size. The following is pseudo-random: "; @@ -135,7 +132,7 @@ namespace AutoClient.Modes.FolderStore { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this); + return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler); } public void SaveChanges() @@ -143,10 +140,5 @@ namespace AutoClient.Modes.FolderStore statusFile.Save(status); changeCounter++; } - - public void OnFailure() - { - failureCount++; - } } } diff --git a/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs new file mode 100644 index 00000000..37d3f7ee --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs @@ -0,0 +1,54 @@ +namespace AutoClient.Modes.FolderStore +{ + public class SlowModeHandler : IFileSaverResultHandler + { + private readonly App app; + private int failureCount = 0; + private bool slowMode = false; + private int recoveryCount = 0; + + public SlowModeHandler(App app) + { + this.app = app; + } + + public void OnSuccess() + { + failureCount = 0; + if (slowMode) + { + recoveryCount++; + if (recoveryCount > 3) + { + Log("Recovery limit reached. Exiting slow mode."); + slowMode = false; + failureCount = 0; + } + } + } + + public void OnFailure() + { + failureCount++; + if (failureCount > 3 && !slowMode) + { + Log("Failure limit reached. Entering slow mode."); + slowMode = true; + recoveryCount = 0; + } + } + + public void Check() + { + if (slowMode) + { + Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes)); + } + } + + private void Log(string msg) + { + app.Log.Log(msg); + } + } +} From 3fe2827080c805acacb8685bff6d14aa46425166 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 2 May 2025 08:49:43 +0200 Subject: [PATCH 3/4] Adds randomness to price per byte per second and contract duration --- Tools/AutoClient/CodexWrapper.cs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Tools/AutoClient/CodexWrapper.cs b/Tools/AutoClient/CodexWrapper.cs index e91c3cfa..02fcca3e 100644 --- a/Tools/AutoClient/CodexWrapper.cs +++ b/Tools/AutoClient/CodexWrapper.cs @@ -7,6 +7,7 @@ namespace AutoClient public class CodexWrapper { private readonly App app; + private static readonly Random r = new Random(); public CodexWrapper(App app, ICodexNode node) { @@ -26,11 +27,11 @@ namespace AutoClient var result = Node.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { CollateralPerByte = app.Config.CollateralPerByte.TstWei(), - Duration = TimeSpan.FromMinutes(app.Config.ContractDurationMinutes), + Duration = GetDuration(), Expiry = TimeSpan.FromMinutes(app.Config.ContractExpiryMinutes), MinRequiredNumberOfNodes = Convert.ToUInt32(app.Config.NumHosts), NodeFailureTolerance = Convert.ToUInt32(app.Config.HostTolerance), - PricePerBytePerSecond = app.Config.PricePerBytePerSecond.TstWei(), + PricePerBytePerSecond = GetPricePerBytePerSecond(), ProofProbability = 15 }); return result; @@ -40,5 +41,25 @@ namespace AutoClient { return Node.GetPurchaseStatus(pid); } + + private TestToken GetPricePerBytePerSecond() + { + var i = app.Config.PricePerBytePerSecond; + i -= 100; + i += r.Next(0, 1000); + + return i.TstWei(); + } + + private TimeSpan GetDuration() + { + var i = app.Config.ContractDurationMinutes; + var day = 60 * 24; + i -= day; + i -= 10; // We don't want to accidentally hit exactly 7 days because that's the limit of the storage node availabilities. + i += r.Next(0, day * 2); + + return TimeSpan.FromMinutes(i); + } } } From a4994f96b8335e33de1eac8f9d06ad2e6ded5730 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Sat, 3 May 2025 08:35:34 +0200 Subject: [PATCH 4/4] Makes chainState fetch requests from chain when it sees events for requests it doesn't already know. --- .../ChainMonitor/ChainState.cs | 24 ++++++++++--------- .../CodexContractsAccess.cs | 15 +++++++++--- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 25a028dc..1bb47912 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -199,21 +199,23 @@ namespace CodexContractsPlugin.ChainMonitor private ChainStateRequest? FindRequest(IHasRequestId request) { var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId)); - if (r == null) + if (r != null) return r; + + try { - var blockNumber = "unknown"; - if (request is IHasBlock blk) - { - blockNumber = blk.Block.BlockNumber.ToString(); - } - - var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " + - $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})"; - + var req = contracts.GetRequest(request.RequestId); + var state = contracts.GetRequestState(req); + var newRequest = new ChainStateRequest(log, req, state); + requests.Add(newRequest); + return newRequest; + } + catch (Exception ex) + { + var msg = "Failed to get request from chain: " + ex; log.Error(msg); handler.OnError(msg); + return null; } - return r; } private bool Equal(byte[] a, byte[] b) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 120574b4..40ee264e 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -3,11 +3,8 @@ using CodexContractsPlugin.Marketplace; using GethPlugin; using Logging; using Nethereum.ABI; -using Nethereum.ABI.FunctionEncoding.Attributes; using Nethereum.Contracts; -using Nethereum.Hex.HexConvertors.Extensions; using Nethereum.Util; -using NethereumWorkflow; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Utils; @@ -28,6 +25,7 @@ namespace CodexContractsPlugin ICodexContractsEvents GetEvents(BlockInterval blockInterval); EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex); RequestState GetRequestState(Request request); + Request GetRequest(byte[] requestId); ulong GetPeriodNumber(DateTime utc); void WaitUntilNextPeriod(); ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period); @@ -126,6 +124,17 @@ namespace CodexContractsPlugin return gethNode.Call(Deployment.MarketplaceAddress, func); } + public Request GetRequest(byte[] requestId) + { + var func = new GetRequestFunction + { + RequestId = requestId + }; + + var request = gethNode.Call(Deployment.MarketplaceAddress, func); + return request.ReturnValue1; + } + public ulong GetPeriodNumber(DateTime utc) { DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);