logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-02-25 07:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Uploads zipfile with archive overview periodically

Browse Source
This commit is contained in:
Ben 2024-11-26 16:15:28 +01:00
parent 6f778ec04f
commit a3b9e7bf8d
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
3 changed files with 77 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Tools/AutoClient/Modes/FolderStore/FileWorker.cs
View File

@ -10,10 +10,11 @@ namespace AutoClient.Modes.FolderStore
private readonly ICodexInstance instance;
private readonly PurchaseInfo purchaseInfo;
private readonly string sourceFilename;
private readonly Action onFileUploaded;
private readonly Action onNewPurchase;
private readonly CodexNode codex;
public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, string filename, Action onNewPurchase)
public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, string filename, Action onFileUploaded, Action onNewPurchase)
: base(app, folder, filename + ".json", purchaseInfo)
{
this.app = app;
@ -21,6 +22,7 @@ namespace AutoClient.Modes.FolderStore
this.instance = instance;
this.purchaseInfo = purchaseInfo;
sourceFilename = filename;
this.onFileUploaded = onFileUploaded;
this.onNewPurchase = onNewPurchase;
codex = new CodexNode(app, instance);
}
@ -75,6 +77,7 @@ namespace AutoClient.Modes.FolderStore
{
Log($"Uploading...");
var cid = await codex.UploadFile(sourceFilename);
onFileUploaded();
Log("Got CID: " + cid);
State.Cid = cid.Id;
Thread.Sleep(1000);

65
Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs
View File

@ -1,4 +1,6 @@
using static AutoClient.Modes.FolderStore.FolderWorkOverview;
using CodexOpenApi;
using System.IO.Compression;
using static AutoClient.Modes.FolderStore.FolderWorkOverview;
namespace AutoClient.Modes.FolderStore
{
@ -15,7 +17,7 @@ namespace AutoClient.Modes.FolderStore
this.purchaseInfo = purchaseInfo;
}
public void Update()
public async Task Update(bool createNewJsonZip, ICodexInstance instance)
{
var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
@ -41,6 +43,65 @@ namespace AutoClient.Modes.FolderStore
State.SuccessfulStored = successful;
State.StoreFailed = failed;
SaveState();
if (createNewJsonZip)
{
await CreateNewOverviewZip(jsonFiles, FilePath, instance);
}
}
private async Task CreateNewOverviewZip(List<string> jsonFiles, string filePath, ICodexInstance instance)
{
Log("");
Log("");
Log("Creating new overview zipfile...");
var zipFilename = CreateZipFile(jsonFiles, filePath);
Log("Uploading to Codex...");
try
{
var codex = new CodexNode(app, instance);
var cid = await codex.UploadFile(zipFilename);
Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
Log("Requesting storage for it...");
var result = await codex.RequestStorage(cid);
Log("Storage requested. Purchase ID: " + result);
}
catch (Exception exc)
{
Log("Failed to upload new overview zipfile: " + exc);
}
Log("");
Log("");
}
private string CreateZipFile(List<string> jsonFiles, string filePath)
{
var zipFilename = Guid.NewGuid().ToString() + ".zip";
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
archive.CreateEntryFromFile(filePath, "overview.json");
foreach (var file in jsonFiles)
{
archive.CreateEntryFromFile(file, Path.GetFileName(file));
}
}
using (var fileStream = new FileStream(zipFilename, FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
return zipFilename;
}
private void Log(string msg)
{
app.Log.Log(msg);
}
[Serializable]

12
Tools/AutoClient/Modes/FolderStoreMode.cs
View File

@ -9,6 +9,7 @@ namespace AutoClient.Modes
private readonly PurchaseInfo purchaseInfo;
private readonly CancellationTokenSource cts = new CancellationTokenSource();
private Task checkTask = Task.CompletedTask;
private int uncommitedChanges;
public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
{
@ -51,7 +52,9 @@ namespace AutoClient.Modes
{
i = 0;
var overview = new FolderWorkOverview(app, purchaseInfo, folder);
overview.Update();
var uploadNewOverview = uncommitedChanges > 10;
await overview.Update(uploadNewOverview, instance);
if (uploadNewOverview) uncommitedChanges = 0;
}
}
}
@ -59,11 +62,16 @@ namespace AutoClient.Modes
private async Task<FileWorker> ProcessWorkItem(ICodexInstance instance)
{
var file = app.FolderWorkDispatcher.GetFileToCheck();
var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnNewPurchase);
var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase);
await worker.Update();
return worker;
}
private void OnFileUploaded()
{
uncommitedChanges++;
}
private void OnNewPurchase()
{
app.FolderWorkDispatcher.ResetIndex();