logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-03 22:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup foldersaver mode

Browse Source
This commit is contained in:
Ben 2025-02-26 16:17:20 +01:00
parent 278b6c95c2
commit f7de11e9c2
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
13 changed files with 359 additions and 728 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ProjectPlugins/CodexClient/StoragePurchaseContract.cs
View File

@ -23,7 +23,7 @@ namespace CodexClient
private readonly ILog log;
private readonly CodexAccess codexAccess;
private readonly ICodexNodeHooks hooks;
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60);
private readonly DateTime contractPendingUtc = DateTime.UtcNow;
private DateTime? contractSubmittedUtc = DateTime.UtcNow;
private DateTime? contractStartedUtc;

80
Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
View File

@ -65,9 +65,9 @@ namespace CodexReleaseTests.MarketTests
var config = GetContracts().Deployment.Config;
foreach (var host in hosts)
{
Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei));
Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei));
AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartHosts));
AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartHosts));
host.Marketplace.MakeStorageAvailable(new StorageAvailability(
totalSpace: HostAvailabilitySize,
maxDuration: HostAvailabilityMaxDuration,
@ -78,22 +78,63 @@ namespace CodexReleaseTests.MarketTests
return hosts;
}
public TestToken GetTstBalance(ICodexNode node)
public void AssertTstBalance(EthAddress address, TestToken expectedBalance, string message)
{
var retry = GetBalanceAssertRetry();
retry.Run(() =>
{
var balance = GetTstBalance(address);
Assert.That(balance, Is.EqualTo(expectedBalance), message);
});
}
public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message)
{
var retry = GetBalanceAssertRetry();
retry.Run(() =>
{
var balance = GetTstBalance(node);
Assert.That(balance, Is.EqualTo(expectedBalance), message);
});
}
public void AssertEthBalance(ICodexNode node, Ether expectedBalance, string message)
{
var retry = GetBalanceAssertRetry();
retry.Run(() =>
{
var balance = GetEthBalance(node);
Assert.That(balance, Is.EqualTo(expectedBalance), message);
});
}
private Retry GetBalanceAssertRetry()
{
return new Retry("AssertBalance",
maxTimeout: TimeSpan.FromMinutes(30.0),
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { });
}
private TestToken GetTstBalance(ICodexNode node)
{
return GetContracts().GetTestTokenBalance(node);
}
public TestToken GetTstBalance(EthAddress address)
private TestToken GetTstBalance(EthAddress address)
{
return GetContracts().GetTestTokenBalance(address);
}
public Ether GetEthBalance(ICodexNode node)
private Ether GetEthBalance(ICodexNode node)
{
return GetGeth().GetEthBalance(node);
}
public Ether GetEthBalance(EthAddress address)
private Ether GetEthBalance(EthAddress address)
{
return GetGeth().GetEthBalance(address);
}
@ -141,10 +182,9 @@ namespace CodexReleaseTests.MarketTests
protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
{
var balance = GetTstBalance(client);
var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts);
Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect.");
AssertTstBalance(client, expectedBalance, "Client balance incorrect.");
}
protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
@ -162,20 +202,10 @@ namespace CodexReleaseTests.MarketTests
expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerBytePerSecond, slotSize, slotDuration);
}
var retry = new Retry(nameof(AssertHostsWerePaidForContract),
maxTimeout: TimeSpan.FromMinutes(30),
sleepAfterFail: TimeSpan.FromSeconds(10),
onFail: f => { }
);
retry.Run(() =>
foreach (var pair in expectedBalances)
{
foreach (var pair in expectedBalances)
{
var balance = GetTstBalance(pair.Key);
Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage.");
}
});
AssertTstBalance(pair.Key, pair.Value, "Host was not paid for storage.");
}
}
protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
@ -184,7 +214,11 @@ namespace CodexReleaseTests.MarketTests
// All host balances should be equal to or greater than the starting balance.
foreach (var host in hosts)
{
Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
var retry = GetBalanceAssertRetry();
retry.Run(() =>
{
Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
});
}
}

10
Tools/AutoClient/App.cs
View File

@ -22,15 +22,6 @@ namespace AutoClient
new ConsoleLog()
));
if (!string.IsNullOrEmpty(config.FolderToStore))
{
FolderWorkDispatcher = new FolderWorkDispatcher(Log, config.FolderToStore);
}
else
{
FolderWorkDispatcher = null!;
}
var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet());
CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath);
@ -41,7 +32,6 @@ namespace AutoClient
public IFileGenerator Generator { get; }
public CancellationTokenSource Cts { get; } = new CancellationTokenSource();
public Performance Performance { get; }
public FolderWorkDispatcher FolderWorkDispatcher { get; }
public CodexNodeFactory CodexNodeFactory { get; }
private IFileGenerator CreateGenerator()

130
Tools/AutoClient/Modes/FolderStore/FileSaver.cs Normal file
View File

@ -0,0 +1,130 @@
using CodexClient;
using Logging;
using Utils;
namespace AutoClient.Modes.FolderStore
{
public class FileSaver
{
private readonly ILog log;
private readonly CodexWrapper instance;
private readonly string folderFile;
private readonly FileStatus entry;
public FileSaver(ILog log, CodexWrapper instance, string folderFile, FileStatus entry)
{
this.log = log;
this.instance = instance;
this.folderFile = folderFile;
this.entry = entry;
}
public bool HasFailed { get; private set; }
public void Process()
{
HasFailed = false;
if (HasRecentPurchase(entry))
{
Log($"Purchase running: '{entry.PurchaseId}'");
return;
}
EnsureBasicCid();
CreateNewPurchase();
}
private void EnsureBasicCid()
{
if (IsBasicCidAvailable()) return;
UploadFile();
}
private bool IsBasicCidAvailable()
{
if (string.IsNullOrEmpty(entry.BasicCid)) return false;
return NodeContainsBasicCid();
}
private bool HasRecentPurchase(FileStatus entry)
{
if (string.IsNullOrEmpty(entry.PurchaseId)) return false;
var purchase = GetPurchase(entry.PurchaseId);
if (purchase == null) return false;
if (!purchase.IsStarted) return false;
// Purchase is started. But, if it finishes soon, we will treat it as already finished.
var threshold = DateTime.UtcNow + TimeSpan.FromHours(3.0);
if (entry.PurchaseFinishedUtc < threshold)
{
Log($"Running purchase will expire soon.");
return false;
}
return true;
}
private StoragePurchase? GetPurchase(string purchaseId)
{
return instance.GetStoragePurchase(purchaseId);
}
private bool NodeContainsBasicCid()
{
try
{
var result = instance.Node.DownloadManifestOnly(new ContentId(entry.BasicCid));
return !string.IsNullOrEmpty(result.Cid.Id);
}
catch
{
Log("Failed to download manifest for basicCid");
return false;
}
}
private void UploadFile()
{
try
{
entry.BasicCid = instance.UploadFile(folderFile).Id;
Log($"Successfully uploaded. BasicCid: '{entry.BasicCid}'");
}
catch (Exception exc)
{
entry.BasicCid = string.Empty;
log.Error("Failed to upload: " + exc);
HasFailed = true;
}
}
private void CreateNewPurchase()
{
if (string.IsNullOrEmpty(entry.BasicCid)) return;
try
{
var request = instance.RequestStorage(new ContentId(entry.BasicCid));
entry.EncodedCid = request.Purchase.ContentId.Id;
entry.PurchaseId = request.PurchaseId;
request.WaitForStorageContractSubmitted();
request.WaitForStorageContractStarted();
entry.PurchaseFinishedUtc = DateTime.UtcNow + request.Purchase.Duration;
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)} ");
}
catch (Exception exc)
{
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
log.Error("Failed to start new purchase: " + exc);
HasFailed = true;
}
}
private void Log(string msg)
{
log.Log(msg);
}
}
}

63
Tools/AutoClient/Modes/FolderStore/FileStatus.cs
View File

@ -1,63 +0,0 @@
using static AutoClient.Modes.FolderStore.FileWorker;
namespace AutoClient.Modes.FolderStore
{
public class FileStatus : JsonBacked<WorkerStatus>
{
private readonly PurchaseInfo purchaseInfo;
public FileStatus(App app, string folder, string filePath, PurchaseInfo purchaseInfo)
: base(app, folder, filePath + ".json")
{
this.purchaseInfo = purchaseInfo;
}
protected override void OnNewState(WorkerStatus newState)
{
newState.LastUpdate = DateTime.MinValue;
}
public bool IsBusy()
{
if (!State.Purchases.Any()) return false;
return State.Purchases.Any(p =>
p.Submitted.HasValue &&
!p.Started.HasValue &&
!p.Expiry.HasValue &&
!p.Finish.HasValue &&
p.Created > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
);
}
public bool IsCurrentlyRunning()
{
if (!State.Purchases.Any()) return false;
return State.Purchases.Any(p =>
p.Submitted.HasValue &&
p.Started.HasValue &&
!p.Expiry.HasValue &&
!p.Finish.HasValue &&
p.Started.Value > DateTime.UtcNow - purchaseInfo.PurchaseDurationTotal
);
}
public bool IsCurrentlyFailed()
{
if (!State.Purchases.Any()) return false;
var mostRecent = GetMostRecent();
if (mostRecent == null) return false;
return mostRecent.Expiry.HasValue;
}
public WorkerPurchase? GetMostRecent()
{
if (!State.Purchases.Any()) return null;
var maxCreated = State.Purchases.Max(p => p.Created);
return State.Purchases.SingleOrDefault(p => p.Created == maxCreated);
}
}
}

311
Tools/AutoClient/Modes/FolderStore/FileWorker.cs
View File

@ -1,311 +0,0 @@
using CodexClient;
using Logging;
namespace AutoClient.Modes.FolderStore
{
public interface IWorkEventHandler
{
void OnFileUploaded();
void OnNewPurchase();
void OnPurchaseExpired();
void OnPurchaseStarted();
}
public class FileWorker
{
private readonly App app;
private readonly CodexWrapper node;
private readonly ILog log;
private readonly PurchaseInfo purchaseInfo;
private readonly string sourceFilename;
private readonly IWorkEventHandler eventHandler;
private readonly FileStatus status;
public FileWorker(App app, CodexWrapper node, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, IWorkEventHandler eventHandler)
{
this.app = app;
this.node = node;
log = new LogPrefixer(app.Log, GetFileTag(fileIndex));
this.purchaseInfo = purchaseInfo;
sourceFilename = fileIndex.File;
if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file.");
this.eventHandler = eventHandler;
status = new FileStatus(app, folder, fileIndex.File, purchaseInfo);
}
public bool IsBusy => status.IsBusy();
public void Update()
{
try
{
if (status.IsCurrentlyRunning() && UpdatedRecently()) return;
Log($"Updating for '{sourceFilename}'...");
EnsureRecentPurchase();
SaveState();
app.Log.Log("");
}
catch (Exception exc)
{
app.Log.Error("Exception during fileworker update: " + exc);
State.Error = exc.ToString();
SaveState();
throw;
}
}
private bool UpdatedRecently()
{
var now = DateTime.UtcNow;
return State.LastUpdate + TimeSpan.FromMinutes(15) > now;
}
private string EnsureCid()
{
Log($"Checking CID...");
if (!string.IsNullOrEmpty(State.EncodedCid) &&
DoesCidExistInNetwork(State.EncodedCid))
{
Log("Encoded-CID successfully found in the network.");
// TODO: Using the encoded CID currently would result in double-encoding of the dataset.
// See: https://github.com/codex-storage/nim-codex/issues/1005
// Always use the basic CID for now, even though we have to repeat the encoding.
// When using encoded CID works: return State.EncodedCid;
}
if (!string.IsNullOrEmpty(State.Cid) &&
DoesCidExistInNetwork(State.Cid))
{
Log("Basic-CID successfully found in the network.");
return State.Cid;
}
if (string.IsNullOrEmpty(State.Cid))
{
Log("File was not previously uploaded.");
}
Log($"Uploading...");
var cid = node.UploadFile(sourceFilename);
eventHandler.OnFileUploaded();
Log("Got Basic-CID: " + cid);
State.Cid = cid.Id;
SaveState();
return State.Cid;
}
private bool DoesCidExistInNetwork(string cid)
{
try
{
// This should not take longer than a few seconds. If it does, cancel it.
var cts = new CancellationTokenSource();
var cancelTask = Task.Run(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(15));
cts.Cancel();
});
var manifest = node.Node.DownloadManifestOnly(new ContentId(cid));
if (manifest == null) return false;
}
catch
{
return false;
}
return true;
}
private void EnsureRecentPurchase()
{
Log($"Checking recent purchase...");
var recent = GetMostRecent();
if (recent == null)
{
Log($"No recent purchase.");
MakeNewPurchase();
return;
}
UpdatePurchase(recent);
if (recent.Expiry.HasValue)
{
Log($"Purchase has failed or expired.");
MakeNewPurchase();
eventHandler.OnPurchaseExpired();
return;
}
if (recent.Finish.HasValue)
{
Log($"Purchase has finished.");
MakeNewPurchase();
return;
}
var safeEnd = recent.Created + purchaseInfo.PurchaseDurationSafe;
if (recent.Started.HasValue && DateTime.UtcNow > safeEnd)
{
Log($"Purchase is going to expire soon.");
MakeNewPurchase();
return;
}
if (!recent.Submitted.HasValue)
{
Log($"Purchase is waiting to be submitted.");
return;
}
if (recent.Submitted.HasValue && !recent.Started.HasValue)
{
Log($"Purchase is submitted and waiting to start.");
return;
}
Log($"Purchase is running.");
}
private void UpdatePurchase(WorkerPurchase recent)
{
if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!");
var now = DateTime.UtcNow;
var purchase = node.GetStoragePurchase(recent.Pid);
if (purchase == null)
{
Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired.");
recent.Expiry = now;
return;
}
if (purchase.IsSubmitted)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
}
if (purchase.IsStarted)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Started.HasValue)
{
Log($"Detected new purchase-start for '{recent.Pid}'.");
recent.Started = now;
eventHandler.OnPurchaseStarted();
}
}
if (purchase.IsCancelled)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Expiry.HasValue) recent.Expiry = now;
}
if (purchase.IsError)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Expiry.HasValue) recent.Expiry = now;
}
if (purchase.IsFinished)
{
if (!recent.Submitted.HasValue) recent.Submitted = now;
if (!recent.Started.HasValue) recent.Started = now;
if (!recent.Finish.HasValue) recent.Finish = now;
}
State.LastUpdate = now;
SaveState();
}
private void MakeNewPurchase()
{
var cid = EnsureCid();
if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!");
Log($"Creating new purchase...");
var response = node.RequestStorage(new ContentId(cid));
var purchaseId = response.PurchaseId;
var encodedCid = response.ContentId;
if (string.IsNullOrEmpty(purchaseId) ||
purchaseId == "Unable to encode manifest" ||
purchaseId == "Purchasing not available" ||
purchaseId == "Expiry required" ||
purchaseId == "Expiry needs to be in future" ||
purchaseId == "Expiry has to be before the request's end (now + duration)")
{
throw new InvalidOperationException(purchaseId);
}
var newPurchase = new WorkerPurchase
{
Created = DateTime.UtcNow,
Pid = purchaseId
};
State.Purchases = State.Purchases.Concat([newPurchase]).ToArray();
State.EncodedCid = encodedCid.Id;
SaveState();
eventHandler.OnNewPurchase();
Log($"New purchase created. PID: '{purchaseId}'.");
Log("Got Encoded-CID: " + encodedCid);
Log("Waiting for submit...");
Thread.Sleep(500);
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
while (DateTime.UtcNow < timeout)
{
Thread.Sleep(5000);
UpdatePurchase(newPurchase);
if (newPurchase.Submitted.HasValue)
{
Log("New purchase successfully submitted.");
return;
}
}
Log("New purchase was not submitted within 5-minute timeout. Will check again later...");
}
private void Log(string msg)
{
log.Log(msg);
}
private WorkerStatus State => status.State;
private void SaveState()
{
status.SaveState();
}
private WorkerPurchase? GetMostRecent()
{
return status.GetMostRecent();
}
private string GetFileTag(FileIndex filename)
{
return $"({filename.Index.ToString("00000")}) ";
}
[Serializable]
public class WorkerStatus
{
public DateTime LastUpdate { get; set; }
public string Cid { get; set; } = string.Empty;
public string EncodedCid { get; set; } = string.Empty;
public string Error { get; set; } = string.Empty;
public WorkerPurchase[] Purchases { get; set; } = Array.Empty<WorkerPurchase>();
}
[Serializable]
public class WorkerPurchase
{
public string Pid { get; set; } = string.Empty;
public DateTime Created { get; set; }
public DateTime? Submitted { get; set; }
public DateTime? Started { get; set; }
public DateTime? Expiry { get; set; }
public DateTime? Finish { get; set; }
}
}
}

96
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs Normal file
View File

@ -0,0 +1,96 @@
using CodexClient;
using Logging;
namespace AutoClient.Modes.FolderStore
{
public class FolderSaver
{
private const string FolderSaverFilename = "foldersaver.json";
private readonly App app;
private readonly CodexWrapper instance;
private readonly JsonFile<FolderStatus> statusFile;
private readonly FolderStatus status;
private int failureCount = 0;
public FolderSaver(App app, CodexWrapper instance)
{
this.app = app;
this.instance = instance;
statusFile = new JsonFile<FolderStatus>(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename));
status = statusFile.Load();
}
public void Run(CancellationTokenSource cts)
{
var folderFiles = Directory.GetFiles(app.Config.FolderToStore);
if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore);
var counter = 0;
foreach (var folderFile in folderFiles)
{
if (cts.IsCancellationRequested) return;
if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename))
{
SaveFile(folderFile);
counter++;
}
if (failureCount > 9)
{
app.Log.Error("Failure count reached threshold. Stopping...");
cts.Cancel();
return;
}
if (counter > 5)
{
counter = 0;
SaveFolderSaverJsonFile();
}
Thread.Sleep(2000);
}
}
private void SaveFile(string folderFile)
{
var localFilename = Path.GetFileName(folderFile);
var entry = status.Files.SingleOrDefault(f => f.Filename == localFilename);
if (entry == null)
{
entry = new FileStatus();
status.Files.Add(entry);
}
ProcessFileEntry(folderFile, entry);
statusFile.Save(status);
}
private void ProcessFileEntry(string folderFile, FileStatus entry)
{
var fileSaver = CreateFileSaver(folderFile, entry);
fileSaver.Process();
if (fileSaver.HasFailed) failureCount++;
}
private void SaveFolderSaverJsonFile()
{
var entry = new FileStatus
{
Filename = FolderSaverFilename
};
var folderFile = Path.Combine(app.Config.FolderToStore, FolderSaverFilename);
var fileSaver = CreateFileSaver(folderFile, entry);
fileSaver.Process();
if (fileSaver.HasFailed) failureCount++;
}
private FileSaver CreateFileSaver(string folderFile, FileStatus entry)
{
var fixedLength = entry.Filename.PadRight(35);
var prefix = $"[{fixedLength}] ";
return new FileSaver(new LogPrefixer(app.Log, prefix), instance, folderFile, entry);
}
}
}

18
Tools/AutoClient/Modes/FolderStore/FolderStatus.cs Normal file
View File

@ -0,0 +1,18 @@
namespace AutoClient.Modes.FolderStore
{
[Serializable]
public class FolderStatus
{
public List<FileStatus> Files { get; set; } = new List<FileStatus>();
}
[Serializable]
public class FileStatus
{
public string Filename { get; set; } = string.Empty;
public string BasicCid { get; set; } = string.Empty;
public string EncodedCid { get; set; } = string.Empty;
public string PurchaseId { get; set; } = string.Empty;
public DateTime PurchaseFinishedUtc { get; set; } = DateTime.MinValue;
}
}

70
Tools/AutoClient/Modes/FolderStore/FolderWorkDispatcher.cs
View File

@ -1,70 +0,0 @@
using Logging;
namespace AutoClient.Modes.FolderStore
{
public class FolderWorkDispatcher
{
private readonly string[] files = Array.Empty<string>();
private readonly ILog log;
private int index = 0;
private int busyCount = 0;
public FolderWorkDispatcher(ILog log, string folder)
{
var fs = Directory.GetFiles(folder);
var result = new List<string>();
foreach (var f in fs)
{
if (!f.ToLowerInvariant().Contains(".json"))
{
var info = new FileInfo(f);
if (info.Exists && info.Length > 1024 * 1024) // larger than 1MB
{
result.Add(f);
}
}
}
files = result.ToArray();
this.log = log;
}
public FileIndex GetFileToCheck()
{
if (busyCount > 0)
{
log.Log("");
log.Log("Max number of busy workers reached. Waiting until contracts are started before creating any more.");
log.Log("");
ResetIndex();
Thread.Sleep(TimeSpan.FromMinutes(1));
}
var file = new FileIndex(files[index], index);
index = (index + 1) % files.Length;
return file;
}
public void ResetIndex()
{
index = 0;
busyCount = 0;
}
public void WorkerIsBusy()
{
busyCount++;
}
}
public class FileIndex
{
public FileIndex(string file, int index)
{
File = file;
Index = index;
}
public string File { get; }
public int Index { get; }
}
}

134
Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs
View File

@ -1,134 +0,0 @@
using System.IO.Compression;
using static AutoClient.Modes.FolderStore.FolderWorkOverview;
namespace AutoClient.Modes.FolderStore
{
public class FolderWorkOverview : JsonBacked<WorkMonitorStatus>
{
private const string OverviewFilename = "codex_folder_saver_overview.json";
private readonly App app;
private readonly PurchaseInfo purchaseInfo;
public FolderWorkOverview(App app, PurchaseInfo purchaseInfo, string folder)
: base(app, folder, Path.Combine(folder, OverviewFilename))
{
this.app = app;
this.purchaseInfo = purchaseInfo;
}
protected override void OnNewState(WorkMonitorStatus newState)
{
newState.LastOverviewUpdate = DateTime.MinValue;
}
public void Update(CodexWrapper instance)
{
var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList();
var total = 0;
var successful = 0;
var failed = 0;
foreach (var file in jsonFiles)
{
try
{
var worker = new FileStatus(app, Folder, file.Substring(0, file.Length - 5), purchaseInfo);
total++;
if (worker.IsCurrentlyRunning()) successful++;
if (worker.IsCurrentlyFailed()) failed++;
}
catch (Exception exc)
{
app.Log.Error("Exception in workoverview update: " + exc);
}
}
State.TotalFiles = total;
State.SuccessfulStored = successful;
State.StoreFailed = failed;
SaveState();
if (State.UncommitedChanges > 3)
{
State.UncommitedChanges = 0;
SaveState();
CreateNewOverviewZip(jsonFiles, FilePath, instance);
}
}
public void MarkUncommitedChange()
{
State.UncommitedChanges++;
SaveState();
}
private void CreateNewOverviewZip(List<string> jsonFiles, string filePath, CodexWrapper node)
{
Log("");
Log("");
Log("Creating new overview zipfile...");
var zipFilename = CreateZipFile(jsonFiles, filePath);
Log("Uploading to Codex...");
try
{
var cid = node.UploadFile(zipFilename);
Log($"Upload successful: New overview zipfile CID = '{cid.Id}'");
Log("Requesting storage for it...");
var result = node.RequestStorage(cid);
Log("Storage requested. Purchase ID: " + result.PurchaseId);
var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid");
File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.ContentId.Id]);
Log($">>> [{outFile}] has been updated. <<<");
}
catch (Exception exc)
{
Log("Failed to upload new overview zipfile: " + exc);
}
Log("");
Log("");
}
private string CreateZipFile(List<string> jsonFiles, string filePath)
{
var zipFilename = Guid.NewGuid().ToString() + ".zip";
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
archive.CreateEntryFromFile(filePath, "overview.json");
foreach (var file in jsonFiles)
{
archive.CreateEntryFromFile(file, Path.GetFileName(file));
}
}
using (var fileStream = new FileStream(zipFilename, FileMode.Create))
{
memoryStream.Seek(0, SeekOrigin.Begin);
memoryStream.CopyTo(fileStream);
}
}
return zipFilename;
}
private void Log(string msg)
{
app.Log.Log(msg);
}
[Serializable]
public class WorkMonitorStatus
{
public int TotalFiles { get; set; }
public int SuccessfulStored { get; set; }
public int StoreFailed { get; set; }
public DateTime LastOverviewUpdate { get; set; }
public int UncommitedChanges { get; set; }
}
}
}

58
Tools/AutoClient/Modes/FolderStore/JsonBacked.cs
View File

@ -1,58 +0,0 @@
using Newtonsoft.Json;
namespace AutoClient.Modes.FolderStore
{
public abstract class JsonBacked<T> where T : new()
{
private readonly App app;
protected JsonBacked(App app, string folder, string filePath)
{
this.app = app;
Folder = folder;
FilePath = filePath;
LoadState();
}
private void LoadState()
{
try
{
if (!File.Exists(FilePath))
{
State = new T();
OnNewState(State);
SaveState();
}
var text = File.ReadAllText(FilePath);
State = JsonConvert.DeserializeObject<T>(text)!;
if (State == null) throw new Exception("Didn't deserialize " + FilePath);
}
catch (Exception exc)
{
app.Log.Error("Failed to load state: " + exc);
}
}
protected string Folder { get; }
protected string FilePath { get; }
public T State { get; private set; } = default!;
protected virtual void OnNewState(T newState)
{
}
public void SaveState()
{
try
{
var json = JsonConvert.SerializeObject(State);
File.WriteAllText(FilePath, json);
}
catch (Exception exc)
{
app.Log.Error("Failed to save state: " + exc);
}
}
}
}

50
Tools/AutoClient/Modes/FolderStore/JsonFile.cs Normal file
View File

@ -0,0 +1,50 @@
using Newtonsoft.Json;
namespace AutoClient.Modes.FolderStore
{
public class JsonFile<T> where T : new()
{
private readonly App app;
private readonly string filePath;
public JsonFile(App app, string filePath)
{
this.app = app;
this.filePath = filePath;
}
public T Load()
{
try
{
if (!File.Exists(filePath))
{
var state = new T();
Save(state);
return state;
}
var text = File.ReadAllText(filePath);
return JsonConvert.DeserializeObject<T>(text)!;
}
catch (Exception exc)
{
app.Log.Error("Failed to load state: " + exc);
throw;
}
}
public void Save(T state)
{
try
{
var json = JsonConvert.SerializeObject(state, Formatting.Indented);
File.WriteAllText(filePath, json);
}
catch (Exception exc)
{
app.Log.Error("Failed to save state: " + exc);
throw;
}
}
}
}

65
Tools/AutoClient/Modes/FolderStoreMode.cs
View File

@ -2,14 +2,13 @@
namespace AutoClient.Modes
{
public class FolderStoreMode : IMode, IWorkEventHandler
public class FolderStoreMode : IMode
{
private readonly App app;
private readonly string folder;
private readonly PurchaseInfo purchaseInfo;
private readonly CancellationTokenSource cts = new CancellationTokenSource();
private Task checkTask = Task.CompletedTask;
private int failureCount = 0;
public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo)
{
@ -24,70 +23,20 @@ namespace AutoClient.Modes
{
try
{
RunChecker(instance);
var saver = new FolderSaver(app, instance);
while (!cts.IsCancellationRequested)
{
saver.Run(cts);
}
}
catch (Exception ex)
{
app.Log.Error("Exception in FolderStoreMode worker: " + ex);
app.Log.Error("Exception in FolderStoreMode: " + ex);
Environment.Exit(1);
}
});
}
private void RunChecker(CodexWrapper instance)
{
var i = 0;
while (!cts.IsCancellationRequested)
{
Thread.Sleep(2000);
var worker = ProcessWorkItem(instance);
if (failureCount > 5)
{
throw new Exception("Failure count > 5. Stopping AutoClient...");
}
i++;
if (i > 5)
{
i = 0;
var overview = new FolderWorkOverview(app, purchaseInfo, folder);
overview.Update(instance);
}
}
}
private FileWorker ProcessWorkItem(CodexWrapper instance)
{
var file = app.FolderWorkDispatcher.GetFileToCheck();
var worker = new FileWorker(app, instance, purchaseInfo, folder, file, this);
worker.Update();
if (worker.IsBusy) app.FolderWorkDispatcher.WorkerIsBusy();
return worker;
}
public void OnFileUploaded()
{
}
public void OnNewPurchase()
{
app.FolderWorkDispatcher.ResetIndex();
var overview = new FolderWorkOverview(app, purchaseInfo, folder);
overview.MarkUncommitedChange();
}
public void OnPurchaseExpired()
{
failureCount++;
}
public void OnPurchaseStarted()
{
failureCount = 0;
}
public void Stop()
{
cts.Cancel();