From a3b9e7bf8db41cc1f435ddbbf2ff2569a3f37c12 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Nov 2024 16:15:28 +0100 Subject: [PATCH] Uploads zipfile with archive overview periodically --- .../Modes/FolderStore/FileWorker.cs | 5 +- .../Modes/FolderStore/FolderWorkOverview.cs | 65 ++++++++++++++++++- Tools/AutoClient/Modes/FolderStoreMode.cs | 12 +++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs b/Tools/AutoClient/Modes/FolderStore/FileWorker.cs index 0cf9bc2a..a18955a8 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileWorker.cs @@ -10,10 +10,11 @@ namespace AutoClient.Modes.FolderStore private readonly ICodexInstance instance; private readonly PurchaseInfo purchaseInfo; private readonly string sourceFilename; + private readonly Action onFileUploaded; private readonly Action onNewPurchase; private readonly CodexNode codex; - public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, string filename, Action onNewPurchase) + public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, string filename, Action onFileUploaded, Action onNewPurchase) : base(app, folder, filename + ".json", purchaseInfo) { this.app = app; @@ -21,6 +22,7 @@ namespace AutoClient.Modes.FolderStore this.instance = instance; this.purchaseInfo = purchaseInfo; sourceFilename = filename; + this.onFileUploaded = onFileUploaded; this.onNewPurchase = onNewPurchase; codex = new CodexNode(app, instance); } @@ -75,6 +77,7 @@ namespace AutoClient.Modes.FolderStore { Log($"Uploading..."); var cid = await codex.UploadFile(sourceFilename); + onFileUploaded(); Log("Got CID: " + cid); State.Cid = cid.Id; Thread.Sleep(1000); diff --git a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs b/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs index 880ae27a..66a50d26 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs @@ -1,4 +1,6 @@ -using static AutoClient.Modes.FolderStore.FolderWorkOverview; +using CodexOpenApi; +using System.IO.Compression; +using static AutoClient.Modes.FolderStore.FolderWorkOverview; namespace AutoClient.Modes.FolderStore { @@ -15,7 +17,7 @@ namespace AutoClient.Modes.FolderStore this.purchaseInfo = purchaseInfo; } - public void Update() + public async Task Update(bool createNewJsonZip, ICodexInstance instance) { var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList(); @@ -41,6 +43,65 @@ namespace AutoClient.Modes.FolderStore State.SuccessfulStored = successful; State.StoreFailed = failed; SaveState(); + + if (createNewJsonZip) + { + await CreateNewOverviewZip(jsonFiles, FilePath, instance); + } + } + + private async Task CreateNewOverviewZip(List jsonFiles, string filePath, ICodexInstance instance) + { + Log(""); + Log(""); + Log("Creating new overview zipfile..."); + var zipFilename = CreateZipFile(jsonFiles, filePath); + + Log("Uploading to Codex..."); + try + { + var codex = new CodexNode(app, instance); + var cid = await codex.UploadFile(zipFilename); + Log($"Upload successful: New overview zipfile CID = '{cid.Id}'"); + Log("Requesting storage for it..."); + var result = await codex.RequestStorage(cid); + Log("Storage requested. Purchase ID: " + result); + } + catch (Exception exc) + { + Log("Failed to upload new overview zipfile: " + exc); + } + Log(""); + Log(""); + } + + private string CreateZipFile(List jsonFiles, string filePath) + { + var zipFilename = Guid.NewGuid().ToString() + ".zip"; + + using (var memoryStream = new MemoryStream()) + { + using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true)) + { + archive.CreateEntryFromFile(filePath, "overview.json"); + foreach (var file in jsonFiles) + { + archive.CreateEntryFromFile(file, Path.GetFileName(file)); + } + } + + using (var fileStream = new FileStream(zipFilename, FileMode.Create)) + { + memoryStream.Seek(0, SeekOrigin.Begin); + memoryStream.CopyTo(fileStream); + } + } + return zipFilename; + } + + private void Log(string msg) + { + app.Log.Log(msg); } [Serializable] diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 68db4c4f..3d0406af 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -9,6 +9,7 @@ namespace AutoClient.Modes private readonly PurchaseInfo purchaseInfo; private readonly CancellationTokenSource cts = new CancellationTokenSource(); private Task checkTask = Task.CompletedTask; + private int uncommitedChanges; public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo) { @@ -51,7 +52,9 @@ namespace AutoClient.Modes { i = 0; var overview = new FolderWorkOverview(app, purchaseInfo, folder); - overview.Update(); + var uploadNewOverview = uncommitedChanges > 10; + await overview.Update(uploadNewOverview, instance); + if (uploadNewOverview) uncommitedChanges = 0; } } } @@ -59,11 +62,16 @@ namespace AutoClient.Modes private async Task ProcessWorkItem(ICodexInstance instance) { var file = app.FolderWorkDispatcher.GetFileToCheck(); - var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnNewPurchase); + var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase); await worker.Update(); return worker; } + private void OnFileUploaded() + { + uncommitedChanges++; + } + private void OnNewPurchase() { app.FolderWorkDispatcher.ResetIndex();