From f7de11e9c2f8dac4e1614c68847c996509895132 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Feb 2025 16:17:20 +0100 Subject: [PATCH] Cleanup foldersaver mode --- .../CodexClient/StoragePurchaseContract.cs | 2 +- .../MarketplaceAutoBootstrapDistTest.cs | 80 +++-- Tools/AutoClient/App.cs | 10 - .../AutoClient/Modes/FolderStore/FileSaver.cs | 130 ++++++++ .../Modes/FolderStore/FileStatus.cs | 63 ---- .../Modes/FolderStore/FileWorker.cs | 311 ------------------ .../Modes/FolderStore/FolderSaver.cs | 96 ++++++ .../Modes/FolderStore/FolderStatus.cs | 18 + .../Modes/FolderStore/FolderWorkDispatcher.cs | 70 ---- .../Modes/FolderStore/FolderWorkOverview.cs | 134 -------- .../Modes/FolderStore/JsonBacked.cs | 58 ---- .../AutoClient/Modes/FolderStore/JsonFile.cs | 50 +++ Tools/AutoClient/Modes/FolderStoreMode.cs | 65 +--- 13 files changed, 359 insertions(+), 728 deletions(-) create mode 100644 Tools/AutoClient/Modes/FolderStore/FileSaver.cs delete mode 100644 Tools/AutoClient/Modes/FolderStore/FileStatus.cs delete mode 100644 Tools/AutoClient/Modes/FolderStore/FileWorker.cs create mode 100644 Tools/AutoClient/Modes/FolderStore/FolderSaver.cs create mode 100644 Tools/AutoClient/Modes/FolderStore/FolderStatus.cs delete mode 100644 Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs delete mode 100644 Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs delete mode 100644 Tools/AutoClient/Modes/FolderStore/JsonBacked.cs create mode 100644 Tools/AutoClient/Modes/FolderStore/JsonFile.cs diff --git a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs index 0455b08f..1ddaff85 100644 --- a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs +++ b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs @@ -23,7 +23,7 @@ namespace CodexClient private readonly ILog log; private readonly CodexAccess codexAccess; private readonly ICodexNodeHooks hooks; - private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30); + private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60); private readonly DateTime contractPendingUtc = DateTime.UtcNow; private DateTime? contractSubmittedUtc = DateTime.UtcNow; private DateTime? contractStartedUtc; diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index a5fa41e7..a53ae95c 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -65,9 +65,9 @@ namespace CodexReleaseTests.MarketTests var config = GetContracts().Deployment.Config; foreach (var host in hosts) { - Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei)); - Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei)); - + AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartHosts)); + AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartHosts)); + host.Marketplace.MakeStorageAvailable(new StorageAvailability( totalSpace: HostAvailabilitySize, maxDuration: HostAvailabilityMaxDuration, @@ -78,22 +78,63 @@ namespace CodexReleaseTests.MarketTests return hosts; } - public TestToken GetTstBalance(ICodexNode node) + public void AssertTstBalance(EthAddress address, TestToken expectedBalance, string message) + { + var retry = GetBalanceAssertRetry(); + retry.Run(() => + { + var balance = GetTstBalance(address); + + Assert.That(balance, Is.EqualTo(expectedBalance), message); + }); + } + + public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message) + { + var retry = GetBalanceAssertRetry(); + retry.Run(() => + { + var balance = GetTstBalance(node); + + Assert.That(balance, Is.EqualTo(expectedBalance), message); + }); + } + + public void AssertEthBalance(ICodexNode node, Ether expectedBalance, string message) + { + var retry = GetBalanceAssertRetry(); + retry.Run(() => + { + var balance = GetEthBalance(node); + + Assert.That(balance, Is.EqualTo(expectedBalance), message); + }); + } + + private Retry GetBalanceAssertRetry() + { + return new Retry("AssertBalance", + maxTimeout: TimeSpan.FromMinutes(30.0), + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }); + } + + private TestToken GetTstBalance(ICodexNode node) { return GetContracts().GetTestTokenBalance(node); } - public TestToken GetTstBalance(EthAddress address) + private TestToken GetTstBalance(EthAddress address) { return GetContracts().GetTestTokenBalance(address); } - public Ether GetEthBalance(ICodexNode node) + private Ether GetEthBalance(ICodexNode node) { return GetGeth().GetEthBalance(node); } - public Ether GetEthBalance(EthAddress address) + private Ether GetEthBalance(EthAddress address) { return GetGeth().GetEthBalance(address); } @@ -141,10 +182,9 @@ namespace CodexReleaseTests.MarketTests protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts) { - var balance = GetTstBalance(client); var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts); - Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect."); + AssertTstBalance(client, expectedBalance, "Client balance incorrect."); } protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts) @@ -162,20 +202,10 @@ namespace CodexReleaseTests.MarketTests expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerBytePerSecond, slotSize, slotDuration); } - var retry = new Retry(nameof(AssertHostsWerePaidForContract), - maxTimeout: TimeSpan.FromMinutes(30), - sleepAfterFail: TimeSpan.FromSeconds(10), - onFail: f => { } - ); - - retry.Run(() => + foreach (var pair in expectedBalances) { - foreach (var pair in expectedBalances) - { - var balance = GetTstBalance(pair.Key); - Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage."); - } - }); + AssertTstBalance(pair.Key, pair.Value, "Host was not paid for storage."); + } } protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts) @@ -184,7 +214,11 @@ namespace CodexReleaseTests.MarketTests // All host balances should be equal to or greater than the starting balance. foreach (var host in hosts) { - Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst())); + var retry = GetBalanceAssertRetry(); + retry.Run(() => + { + Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst())); + }); } } diff --git a/Tools/AutoClient/App.cs b/Tools/AutoClient/App.cs index 215728e5..74839c40 100644 --- a/Tools/AutoClient/App.cs +++ b/Tools/AutoClient/App.cs @@ -22,15 +22,6 @@ namespace AutoClient new ConsoleLog() )); - if (!string.IsNullOrEmpty(config.FolderToStore)) - { - FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore); - } - else - { - FolderWorkDispatcher = null!; - } - var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet()); CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath); @@ -41,7 +32,6 @@ namespace AutoClient public IFileGenerator Generator { get; } public CancellationTokenSource Cts { get; } = new CancellationTokenSource(); public Performance Performance { get; } - public FolderWorkDispatcher FolderWorkDispatcher { get; } public CodexNodeFactory CodexNodeFactory { get; } private IFileGenerator CreateGenerator() diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs new file mode 100644 index 00000000..dad87d89 --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -0,0 +1,130 @@ +using CodexClient; +using Logging; +using Utils; + +namespace AutoClient.Modes.FolderStore +{ + public class FileSaver + { + private readonly ILog log; + private readonly CodexWrapper instance; + private readonly string folderFile; + private readonly FileStatus entry; + + public FileSaver(ILog log, CodexWrapper instance, string folderFile, FileStatus entry) + { + this.log = log; + this.instance = instance; + this.folderFile = folderFile; + this.entry = entry; + } + + public bool HasFailed { get; private set; } + + public void Process() + { + HasFailed = false; + if (HasRecentPurchase(entry)) + { + Log($"Purchase running: '{entry.PurchaseId}'"); + return; + } + + EnsureBasicCid(); + CreateNewPurchase(); + } + + private void EnsureBasicCid() + { + if (IsBasicCidAvailable()) return; + UploadFile(); + } + + private bool IsBasicCidAvailable() + { + if (string.IsNullOrEmpty(entry.BasicCid)) return false; + return NodeContainsBasicCid(); + } + + private bool HasRecentPurchase(FileStatus entry) + { + if (string.IsNullOrEmpty(entry.PurchaseId)) return false; + var purchase = GetPurchase(entry.PurchaseId); + if (purchase == null) return false; + if (!purchase.IsStarted) return false; + + // Purchase is started. But, if it finishes soon, we will treat it as already finished. + var threshold = DateTime.UtcNow + TimeSpan.FromHours(3.0); + if (entry.PurchaseFinishedUtc < threshold) + { + Log($"Running purchase will expire soon."); + return false; + } + return true; + } + + private StoragePurchase? GetPurchase(string purchaseId) + { + return instance.GetStoragePurchase(purchaseId); + } + + private bool NodeContainsBasicCid() + { + try + { + var result = instance.Node.DownloadManifestOnly(new ContentId(entry.BasicCid)); + return !string.IsNullOrEmpty(result.Cid.Id); + } + catch + { + Log("Failed to download manifest for basicCid"); + return false; + } + } + + private void UploadFile() + { + try + { + entry.BasicCid = instance.UploadFile(folderFile).Id; + Log($"Successfully uploaded. BasicCid: '{entry.BasicCid}'"); + } + catch (Exception exc) + { + entry.BasicCid = string.Empty; + log.Error("Failed to upload: " + exc); + HasFailed = true; + } + } + + private void CreateNewPurchase() + { + if (string.IsNullOrEmpty(entry.BasicCid)) return; + + try + { + var request = instance.RequestStorage(new ContentId(entry.BasicCid)); + entry.EncodedCid = request.Purchase.ContentId.Id; + entry.PurchaseId = request.PurchaseId; + + request.WaitForStorageContractSubmitted(); + request.WaitForStorageContractStarted(); + + entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration; + Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)} "); + } + catch (Exception exc) + { + entry.EncodedCid = string.Empty; + entry.PurchaseId = string.Empty; + log.Error("Failed to start new purchase: " + exc); + HasFailed = true; + } + } + + private void Log(string msg) + { + log.Log(msg); + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/FileStatus.cs b/Tools/AutoClient/Modes/FolderStore/FileStatus.cs deleted file mode 100644 index c9c1ec0e..00000000 --- a/Tools/AutoClient/Modes/FolderStore/FileStatus.cs +++ /dev/null @@ -1,63 +0,0 @@ -using static AutoClient.Modes.FolderStore.FileWorker; - -namespace AutoClient.Modes.FolderStore -{ - public class FileStatus : JsonBacked - { - private readonly PurchaseInfo purchaseInfo; - - public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo) - : base(app, folder, filePath + ".json") - { - this.purchaseInfo = purchaseInfo; - } - - protected override void OnNewState(WorkerStatus newState) - { - newState.LastUpdate = DateTime.MinValue; - } - - public bool IsBusy() - { - if (!State.Purchases.Any()) return false; - - return State.Purchases.Any(p => - p.Submitted.HasValue && - !p.Started.HasValue && - !p.Expiry.HasValue && - !p.Finish.HasValue && - p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal - ); - } - - public bool IsCurrentlyRunning() - { - if (!State.Purchases.Any()) return false; - - return State.Purchases.Any(p => - p.Submitted.HasValue && - p.Started.HasValue && - !p.Expiry.HasValue && - !p.Finish.HasValue && - p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal - ); - } - - public bool IsCurrentlyFailed() - { - if (!State.Purchases.Any()) return false; - - var mostRecent = GetMostRecent(); - if (mostRecent == null) return false; - - return mostRecent.Expiry.HasValue; - } - - public WorkerPurchase? GetMostRecent() - { - if (!State.Purchases.Any()) return null; - var maxCreated = State.Purchases.Max(p => p.Created); - return State.Purchases.SingleOrDefault(p => p.Created == maxCreated); - } - } -} diff --git a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs b/Tools/AutoClient/Modes/FolderStore/FileWorker.cs deleted file mode 100644 index c5783de7..00000000 --- a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs +++ /dev/null @@ -1,311 +0,0 @@ -using CodexClient; -using Logging; - -namespace AutoClient.Modes.FolderStore -{ - public interface IWorkEventHandler - { - void OnFileUploaded(); - void OnNewPurchase(); - void OnPurchaseExpired(); - void OnPurchaseStarted(); - } - - public class FileWorker - { - private readonly App app; - private readonly CodexWrapper node; - private readonly ILog log; - private readonly PurchaseInfo purchaseInfo; - private readonly string sourceFilename; - private readonly IWorkEventHandler eventHandler; - private readonly FileStatus status; - - public FileWorker(App app, CodexWrapper node, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, IWorkEventHandler eventHandler) - { - this.app = app; - this.node = node; - log = new LogPrefixer(app.Log, GetFileTag(fileIndex)); - this.purchaseInfo = purchaseInfo; - sourceFilename = fileIndex.File; - if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file."); - this.eventHandler = eventHandler; - - status = new FileStatus(app, folder, fileIndex.File, purchaseInfo); - } - - public bool IsBusy => status.IsBusy(); - - public void Update() - { - try - { - if (status.IsCurrentlyRunning() && UpdatedRecently()) return; - - Log($"Updating for '{sourceFilename}'..."); - EnsureRecentPurchase(); - SaveState(); - app.Log.Log(""); - } - catch (Exception exc) - { - app.Log.Error("Exception during fileworker update: " + exc); - State.Error = exc.ToString(); - SaveState(); - throw; - } - } - - private bool UpdatedRecently() - { - var now = DateTime.UtcNow; - return State.LastUpdate + TimeSpan.FromMinutes(15) > now; - } - - private string EnsureCid() - { - Log($"Checking CID..."); - - if (!string.IsNullOrEmpty(State.EncodedCid) && - DoesCidExistInNetwork(State.EncodedCid)) - { - Log("Encoded-CID successfully found in the network."); - // TODO: Using the encoded CID currently would result in double-encoding of the dataset. - // See: https://github.com/codex-storage/nim-codex/issues/1005 - // Always use the basic CID for now, even though we have to repeat the encoding. - // When using encoded CID works: return State.EncodedCid; - } - - if (!string.IsNullOrEmpty(State.Cid) && - DoesCidExistInNetwork(State.Cid)) - { - Log("Basic-CID successfully found in the network."); - return State.Cid; - } - - if (string.IsNullOrEmpty(State.Cid)) - { - Log("File was not previously uploaded."); - } - - Log($"Uploading..."); - var cid = node.UploadFile(sourceFilename); - eventHandler.OnFileUploaded(); - Log("Got Basic-CID: " + cid); - State.Cid = cid.Id; - SaveState(); - return State.Cid; - } - - private bool DoesCidExistInNetwork(string cid) - { - try - { - // This should not take longer than a few seconds. If it does, cancel it. - var cts = new CancellationTokenSource(); - var cancelTask = Task.Run(() => - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - cts.Cancel(); - }); - - var manifest = node.Node.DownloadManifestOnly(new ContentId(cid)); - if (manifest == null) return false; - } - catch - { - return false; - } - return true; - } - - private void EnsureRecentPurchase() - { - Log($"Checking recent purchase..."); - var recent = GetMostRecent(); - if (recent == null) - { - Log($"No recent purchase."); - MakeNewPurchase(); - return; - } - - UpdatePurchase(recent); - - if (recent.Expiry.HasValue) - { - Log($"Purchase has failed or expired."); - MakeNewPurchase(); - eventHandler.OnPurchaseExpired(); - return; - } - - if (recent.Finish.HasValue) - { - Log($"Purchase has finished."); - MakeNewPurchase(); - return; - } - - var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe; - if (recent.Started.HasValue && DateTime.UtcNow > safeEnd) - { - Log($"Purchase is going to expire soon."); - MakeNewPurchase(); - return; - } - - if (!recent.Submitted.HasValue) - { - Log($"Purchase is waiting to be submitted."); - return; - } - - if (recent.Submitted.HasValue && !recent.Started.HasValue) - { - Log($"Purchase is submitted and waiting to start."); - return; - } - - Log($"Purchase is running."); - } - - private void UpdatePurchase(WorkerPurchase recent) - { - if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!"); - var now = DateTime.UtcNow; - - var purchase = node.GetStoragePurchase(recent.Pid); - if (purchase == null) - { - Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired."); - recent.Expiry = now; - return; - } - - if (purchase.IsSubmitted) - { - if (!recent.Submitted.HasValue) recent.Submitted = now; - } - if (purchase.IsStarted) - { - if (!recent.Submitted.HasValue) recent.Submitted = now; - if (!recent.Started.HasValue) - { - Log($"Detected new purchase-start for '{recent.Pid}'."); - recent.Started = now; - eventHandler.OnPurchaseStarted(); - } - } - if (purchase.IsCancelled) - { - if (!recent.Submitted.HasValue) recent.Submitted = now; - if (!recent.Expiry.HasValue) recent.Expiry = now; - } - if (purchase.IsError) - { - if (!recent.Submitted.HasValue) recent.Submitted = now; - if (!recent.Expiry.HasValue) recent.Expiry = now; - } - if (purchase.IsFinished) - { - if (!recent.Submitted.HasValue) recent.Submitted = now; - if (!recent.Started.HasValue) recent.Started = now; - if (!recent.Finish.HasValue) recent.Finish = now; - } - State.LastUpdate = now; - SaveState(); - } - - private void MakeNewPurchase() - { - var cid = EnsureCid(); - if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!"); - - Log($"Creating new purchase..."); - var response = node.RequestStorage(new ContentId(cid)); - var purchaseId = response.PurchaseId; - var encodedCid = response.ContentId; - if (string.IsNullOrEmpty(purchaseId) || - purchaseId == "Unable to encode manifest" || - purchaseId == "Purchasing not available" || - purchaseId == "Expiry required" || - purchaseId == "Expiry needs to be in future" || - purchaseId == "Expiry has to be before the request's end (now + duration)") - { - throw new InvalidOperationException(purchaseId); - } - - var newPurchase = new WorkerPurchase - { - Created = DateTime.UtcNow, - Pid = purchaseId - }; - State.Purchases = State.Purchases.Concat([newPurchase]).ToArray(); - State.EncodedCid = encodedCid.Id; - SaveState(); - eventHandler.OnNewPurchase(); - - Log($"New purchase created. PID: '{purchaseId}'."); - Log("Got Encoded-CID: " + encodedCid); - Log("Waiting for submit..."); - Thread.Sleep(500); - - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); - while (DateTime.UtcNow < timeout) - { - Thread.Sleep(5000); - UpdatePurchase(newPurchase); - if (newPurchase.Submitted.HasValue) - { - Log("New purchase successfully submitted."); - return; - } - } - Log("New purchase was not submitted within 5-minute timeout. Will check again later..."); - } - - private void Log(string msg) - { - log.Log(msg); - } - - private WorkerStatus State => status.State; - - private void SaveState() - { - status.SaveState(); - } - - private WorkerPurchase? GetMostRecent() - { - return status.GetMostRecent(); - } - - private string GetFileTag(FileIndex filename) - { - return $"({filename.Index.ToString("00000")}) "; - } - - [Serializable] - public class WorkerStatus - { - public DateTime LastUpdate { get; set; } - public string Cid { get; set; } = string.Empty; - public string EncodedCid { get; set; } = string.Empty; - public string Error { get; set; } = string.Empty; - public WorkerPurchase[] Purchases { get; set; } = Array.Empty(); - } - - [Serializable] - public class WorkerPurchase - { - public string Pid { get; set; } = string.Empty; - public DateTime Created { get; set; } - public DateTime? Submitted { get; set; } - public DateTime? Started { get; set; } - public DateTime? Expiry { get; set; } - public DateTime? Finish { get; set; } - } - } -} diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs new file mode 100644 index 00000000..16dde6d3 --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -0,0 +1,96 @@ +using CodexClient; +using Logging; + +namespace AutoClient.Modes.FolderStore +{ + public class FolderSaver + { + private const string FolderSaverFilename = "foldersaver.json"; + private readonly App app; + private readonly CodexWrapper instance; + private readonly JsonFile statusFile; + private readonly FolderStatus status; + private int failureCount = 0; + + public FolderSaver(App app, CodexWrapper instance) + { + this.app = app; + this.instance = instance; + + statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); + status = statusFile.Load(); + } + + public void Run(CancellationTokenSource cts) + { + var folderFiles = Directory.GetFiles(app.Config.FolderToStore); + if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); + + var counter = 0; + foreach (var folderFile in folderFiles) + { + if (cts.IsCancellationRequested) return; + + if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename)) + { + SaveFile(folderFile); + counter++; + } + + if (failureCount > 9) + { + app.Log.Error("Failure count reached threshold. Stopping..."); + cts.Cancel(); + return; + } + + if (counter > 5) + { + counter = 0; + SaveFolderSaverJsonFile(); + } + + Thread.Sleep(2000); + } + } + + private void SaveFile(string folderFile) + { + var localFilename = Path.GetFileName(folderFile); + var entry = status.Files.SingleOrDefault(f => f.Filename == localFilename); + if (entry == null) + { + entry = new FileStatus(); + status.Files.Add(entry); + } + ProcessFileEntry(folderFile, entry); + statusFile.Save(status); + } + + private void ProcessFileEntry(string folderFile, FileStatus entry) + { + var fileSaver = CreateFileSaver(folderFile, entry); + fileSaver.Process(); + if (fileSaver.HasFailed) failureCount++; + } + + private void SaveFolderSaverJsonFile() + { + var entry = new FileStatus + { + Filename = FolderSaverFilename + }; + var folderFile = Path.Combine(app.Config.FolderToStore, FolderSaverFilename); + var fileSaver = CreateFileSaver(folderFile, entry); + fileSaver.Process(); + if (fileSaver.HasFailed) failureCount++; + } + + private FileSaver CreateFileSaver(string folderFile, FileStatus entry) + { + var fixedLength = entry.Filename.PadRight(35); + var prefix = $"[{fixedLength}] "; + return new FileSaver(new LogPrefixer(app.Log, prefix), instance, folderFile, entry); + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs new file mode 100644 index 00000000..9b68b7ce --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs @@ -0,0 +1,18 @@ +namespace AutoClient.Modes.FolderStore +{ + [Serializable] + public class FolderStatus + { + public List Files { get; set; } = new List(); + } + + [Serializable] + public class FileStatus + { + public string Filename { get; set; } = string.Empty; + public string BasicCid { get; set; } = string.Empty; + public string EncodedCid { get; set; } = string.Empty; + public string PurchaseId { get; set; } = string.Empty; + public DateTime PurchaseFinishedUtc { get; set; } = DateTime.MinValue; + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs b/Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs deleted file mode 100644 index 5067ccee..00000000 --- a/Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs +++ /dev/null @@ -1,70 +0,0 @@ -using Logging; - -namespace AutoClient.Modes.FolderStore -{ - public class FolderWorkDispatcher - { - private readonly string[] files = Array.Empty(); - private readonly ILog log; - private int index = 0; - private int busyCount = 0; - - public FolderWorkDispatcher(ILog log, string folder) - { - var fs = Directory.GetFiles(folder); - var result = new List(); - foreach (var f in fs) - { - if (!f.ToLowerInvariant().Contains(".json")) - { - var info = new FileInfo(f); - if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB - { - result.Add(f); - } - } - } - files = result.ToArray(); - this.log = log; - } - - public FileIndex GetFileToCheck() - { - if (busyCount > 0) - { - log.Log(""); - log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more."); - log.Log(""); - ResetIndex(); - Thread.Sleep(TimeSpan.FromMinutes(1)); - } - - var file = new FileIndex(files[index], index); - index = (index + 1) % files.Length; - return file; - } - - public void ResetIndex() - { - index = 0; - busyCount = 0; - } - - public void WorkerIsBusy() - { - busyCount++; - } - } - - public class FileIndex - { - public FileIndex(string file, int index) - { - File = file; - Index = index; - } - - public string File { get; } - public int Index { get; } - } -} diff --git a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs b/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs deleted file mode 100644 index aadfe037..00000000 --- a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs +++ /dev/null @@ -1,134 +0,0 @@ -using System.IO.Compression; -using static AutoClient.Modes.FolderStore.FolderWorkOverview; - -namespace AutoClient.Modes.FolderStore -{ - public class FolderWorkOverview : JsonBacked - { - private const string OverviewFilename = "codex_folder_saver_overview.json"; - private readonly App app; - private readonly PurchaseInfo purchaseInfo; - - public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder) - : base(app, folder, Path.Combine(folder, OverviewFilename)) - { - this.app = app; - this.purchaseInfo = purchaseInfo; - } - - protected override void OnNewState(WorkMonitorStatus newState) - { - newState.LastOverviewUpdate = DateTime.MinValue; - } - - public void Update(CodexWrapper instance) - { - var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList(); - - var total = 0; - var successful = 0; - var failed = 0; - foreach (var file in jsonFiles) - { - try - { - var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo); - total++; - if (worker.IsCurrentlyRunning()) successful++; - if (worker.IsCurrentlyFailed()) failed++; - } - catch (Exception exc) - { - app.Log.Error("Exception in workoverview update: " + exc); - } - } - - State.TotalFiles = total; - State.SuccessfulStored = successful; - State.StoreFailed = failed; - SaveState(); - - if (State.UncommitedChanges > 3) - { - State.UncommitedChanges = 0; - SaveState(); - - CreateNewOverviewZip(jsonFiles, FilePath, instance); - } - } - - public void MarkUncommitedChange() - { - State.UncommitedChanges++; - SaveState(); - } - - private void CreateNewOverviewZip(List jsonFiles, string filePath, CodexWrapper node) - { - Log(""); - Log(""); - Log("Creating new overview zipfile..."); - var zipFilename = CreateZipFile(jsonFiles, filePath); - - Log("Uploading to Codex..."); - try - { - var cid = node.UploadFile(zipFilename); - Log($"Upload successful: New overview zipfile CID = '{cid.Id}'"); - Log("Requesting storage for it..."); - var result = node.RequestStorage(cid); - Log("Storage requested. Purchase ID: " + result.PurchaseId); - - var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid"); - File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.ContentId.Id]); - Log($">>> [{outFile}] has been updated. <<<"); - } - catch (Exception exc) - { - Log("Failed to upload new overview zipfile: " + exc); - } - Log(""); - Log(""); - } - - private string CreateZipFile(List jsonFiles, string filePath) - { - var zipFilename = Guid.NewGuid().ToString() + ".zip"; - - using (var memoryStream = new MemoryStream()) - { - using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true)) - { - archive.CreateEntryFromFile(filePath, "overview.json"); - foreach (var file in jsonFiles) - { - archive.CreateEntryFromFile(file, Path.GetFileName(file)); - } - } - - using (var fileStream = new FileStream(zipFilename, FileMode.Create)) - { - memoryStream.Seek(0, SeekOrigin.Begin); - memoryStream.CopyTo(fileStream); - } - } - return zipFilename; - } - - private void Log(string msg) - { - app.Log.Log(msg); - } - - [Serializable] - public class WorkMonitorStatus - { - public int TotalFiles { get; set; } - public int SuccessfulStored { get; set; } - public int StoreFailed { get; set; } - - public DateTime LastOverviewUpdate { get; set; } - public int UncommitedChanges { get; set; } - } - } -} diff --git a/Tools/AutoClient/Modes/FolderStore/JsonBacked.cs b/Tools/AutoClient/Modes/FolderStore/JsonBacked.cs deleted file mode 100644 index 8bfa63cd..00000000 --- a/Tools/AutoClient/Modes/FolderStore/JsonBacked.cs +++ /dev/null @@ -1,58 +0,0 @@ -using Newtonsoft.Json; - -namespace AutoClient.Modes.FolderStore -{ - public abstract class JsonBacked where T : new() - { - private readonly App app; - - protected JsonBacked(App app, string folder, string filePath) - { - this.app = app; - Folder = folder; - FilePath = filePath; - LoadState(); - } - - private void LoadState() - { - try - { - if (!File.Exists(FilePath)) - { - State = new T(); - OnNewState(State); - SaveState(); - } - var text = File.ReadAllText(FilePath); - State = JsonConvert.DeserializeObject(text)!; - if (State == null) throw new Exception("Didn't deserialize " + FilePath); - } - catch (Exception exc) - { - app.Log.Error("Failed to load state: " + exc); - } - } - - protected string Folder { get; } - protected string FilePath { get; } - public T State { get; private set; } = default!; - - protected virtual void OnNewState(T newState) - { - } - - public void SaveState() - { - try - { - var json = JsonConvert.SerializeObject(State); - File.WriteAllText(FilePath, json); - } - catch (Exception exc) - { - app.Log.Error("Failed to save state: " + exc); - } - } - } -} diff --git a/Tools/AutoClient/Modes/FolderStore/JsonFile.cs b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs new file mode 100644 index 00000000..6d0f972d --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs @@ -0,0 +1,50 @@ +using Newtonsoft.Json; + +namespace AutoClient.Modes.FolderStore +{ + public class JsonFile where T : new() + { + private readonly App app; + private readonly string filePath; + + public JsonFile(App app, string filePath) + { + this.app = app; + this.filePath = filePath; + } + + public T Load() + { + try + { + if (!File.Exists(filePath)) + { + var state = new T(); + Save(state); + return state; + } + var text = File.ReadAllText(filePath); + return JsonConvert.DeserializeObject(text)!; + } + catch (Exception exc) + { + app.Log.Error("Failed to load state: " + exc); + throw; + } + } + + public void Save(T state) + { + try + { + var json = JsonConvert.SerializeObject(state, Formatting.Indented); + File.WriteAllText(filePath, json); + } + catch (Exception exc) + { + app.Log.Error("Failed to save state: " + exc); + throw; + } + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 04a6965d..477ea9bf 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -2,14 +2,13 @@ namespace AutoClient.Modes { - public class FolderStoreMode : IMode, IWorkEventHandler + public class FolderStoreMode : IMode { private readonly App app; private readonly string folder; private readonly PurchaseInfo purchaseInfo; private readonly CancellationTokenSource cts = new CancellationTokenSource(); private Task checkTask = Task.CompletedTask; - private int failureCount = 0; public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo) { @@ -24,70 +23,20 @@ namespace AutoClient.Modes { try { - RunChecker(instance); + var saver = new FolderSaver(app, instance); + while (!cts.IsCancellationRequested) + { + saver.Run(cts); + } } catch (Exception ex) { - app.Log.Error("Exception in FolderStoreMode worker: " + ex); + app.Log.Error("Exception in FolderStoreMode: " + ex); Environment.Exit(1); } }); } - private void RunChecker(CodexWrapper instance) - { - var i = 0; - while (!cts.IsCancellationRequested) - { - Thread.Sleep(2000); - - var worker = ProcessWorkItem(instance); - if (failureCount > 5) - { - throw new Exception("Failure count > 5. Stopping AutoClient..."); - } - i++; - - if (i > 5) - { - i = 0; - var overview = new FolderWorkOverview(app, purchaseInfo, folder); - overview.Update(instance); - } - } - } - - private FileWorker ProcessWorkItem(CodexWrapper instance) - { - var file = app.FolderWorkDispatcher.GetFileToCheck(); - var worker = new FileWorker(app, instance, purchaseInfo, folder, file, this); - worker.Update(); - if (worker.IsBusy) app.FolderWorkDispatcher.WorkerIsBusy(); - return worker; - } - - public void OnFileUploaded() - { - } - - public void OnNewPurchase() - { - app.FolderWorkDispatcher.ResetIndex(); - - var overview = new FolderWorkOverview(app, purchaseInfo, folder); - overview.MarkUncommitedChange(); - } - - public void OnPurchaseExpired() - { - failureCount++; - } - - public void OnPurchaseStarted() - { - failureCount = 0; - } - public void Stop() { cts.Cancel();