logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-06-02 06:29:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/extended-marketplace-testing

Browse Source
This commit is contained in:
ThatBen 2025-05-03 08:41:37 +02:00
commit 477aadb726
No known key found for this signature in database
GPG Key ID: E020A7DDCD52E1AB
9 changed files with 163 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -199,21 +199,23 @@ namespace CodexContractsPlugin.ChainMonitor
private ChainStateRequest? FindRequest(IHasRequestId request)
{
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
if (r == null)
if (r != null) return r;
try
{
var blockNumber = "unknown";
if (request is IHasBlock blk)
{
blockNumber = blk.Block.BlockNumber.ToString();
}
var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
$"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
var req = contracts.GetRequest(request.RequestId);
var state = contracts.GetRequestState(req);
var newRequest = new ChainStateRequest(log, req, state);
requests.Add(newRequest);
return newRequest;
}
catch (Exception ex)
{
var msg = "Failed to get request from chain: " + ex;
log.Error(msg);
handler.OnError(msg);
return null;
}
return r;
}
private bool Equal(byte[] a, byte[] b)

15
ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs
View File

@ -3,11 +3,8 @@ using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.ABI;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Util;
using NethereumWorkflow;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Utils;
@ -28,6 +25,7 @@ namespace CodexContractsPlugin
ICodexContractsEvents GetEvents(BlockInterval blockInterval);
EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
RequestState GetRequestState(Request request);
Request GetRequest(byte[] requestId);
ulong GetPeriodNumber(DateTime utc);
void WaitUntilNextPeriod();
ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
@ -126,6 +124,17 @@ namespace CodexContractsPlugin
return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
}
public Request GetRequest(byte[] requestId)
{
var func = new GetRequestFunction
{
RequestId = requestId
};
var request = gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(Deployment.MarketplaceAddress, func);
return request.ReturnValue1;
}
public ulong GetPeriodNumber(DateTime utc)
{
DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);

25
Tools/AutoClient/CodexWrapper.cs
View File

@ -7,6 +7,7 @@ namespace AutoClient
public class CodexWrapper
{
private readonly App app;
private static readonly Random r = new Random();
public CodexWrapper(App app, ICodexNode node)
{
@ -26,11 +27,11 @@ namespace AutoClient
var result = Node.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
CollateralPerByte = app.Config.CollateralPerByte.TstWei(),
Duration = TimeSpan.FromMinutes(app.Config.ContractDurationMinutes),
Duration = GetDuration(),
Expiry = TimeSpan.FromMinutes(app.Config.ContractExpiryMinutes),
MinRequiredNumberOfNodes = Convert.ToUInt32(app.Config.NumHosts),
NodeFailureTolerance = Convert.ToUInt32(app.Config.HostTolerance),
PricePerBytePerSecond = app.Config.PricePerBytePerSecond.TstWei(),
PricePerBytePerSecond = GetPricePerBytePerSecond(),
ProofProbability = 15
});
return result;
@ -40,5 +41,25 @@ namespace AutoClient
{
return Node.GetPurchaseStatus(pid);
}
private TestToken GetPricePerBytePerSecond()
{
var i = app.Config.PricePerBytePerSecond;
i -= 100;
i += r.Next(0, 1000);
return i.TstWei();
}
private TimeSpan GetDuration()
{
var i = app.Config.ContractDurationMinutes;
var day = 60 * 24;
i -= day;
i -= 10; // We don't want to accidentally hit exactly 7 days because that's the limit of the storage node availabilities.
i += r.Next(0, day * 2);
return TimeSpan.FromMinutes(i);
}
}
}

3
Tools/AutoClient/Configuration.cs
View File

@ -59,6 +59,9 @@ namespace AutoClient
"/root/codex-testnet-starter/scripts/eth_7.address" + ";" +
"/root/codex-testnet-starter/scripts/eth_8.address";
[Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")]
public int SlowModeDelayMinutes { get; set; } = 60 * 1;
public string LogPath
{
get

46
Tools/AutoClient/Modes/FolderStore/FileSaver.cs
View File

@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore
public interface IFileSaverEventHandler
{
void SaveChanges();
}
public interface IFileSaverResultHandler
{
void OnSuccess();
void OnFailure();
}
@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
private readonly IFileSaverEventHandler handler;
private readonly IFileSaverEventHandler saveHandler;
private readonly IFileSaverResultHandler resultHandler;
public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
{
this.log = log;
this.loadBalancer = loadBalancer;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
this.handler = handler;
this.saveHandler = saveHandler;
this.resultHandler = resultHandler;
}
public void Process()
@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore
loadBalancer.DispatchOnCodex(instance =>
{
entry.CodexNodeId = instance.Node.GetName();
handler.SaveChanges();
saveHandler.SaveChanges();
var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
run.Process();
});
}
@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore
{
loadBalancer.DispatchOnSpecificCodex(instance =>
{
var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler);
var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler);
run.Process();
}, entry.CodexNodeId);
}
@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore
private readonly Stats stats;
private readonly string folderFile;
private readonly FileStatus entry;
private readonly IFileSaverEventHandler handler;
private readonly IFileSaverEventHandler saveHandler;
private readonly IFileSaverResultHandler resultHandler;
private readonly QuotaCheck quotaCheck;
public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler)
public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler)
{
this.log = log;
this.instance = instance;
this.stats = stats;
this.folderFile = folderFile;
this.entry = entry;
this.handler = handler;
this.saveHandler = saveHandler;
this.resultHandler = resultHandler;
quotaCheck = new QuotaCheck(log, folderFile, instance);
}
@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore
Thread.Sleep(TimeSpan.FromMinutes(1.0));
}
Log("Could not upload: Insufficient local storage quota.");
handler.OnFailure();
resultHandler.OnFailure();
return false;
}
@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore
entry.BasicCid = string.Empty;
stats.FailedUploads++;
log.Error("Failed to upload: " + exc);
handler.OnFailure();
resultHandler.OnFailure();
}
handler.SaveChanges();
saveHandler.SaveChanges();
}
private void CreateNewPurchase()
@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore
WaitForStarted(request);
stats.StorageRequestStats.SuccessfullyStarted++;
handler.SaveChanges();
saveHandler.SaveChanges();
Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}");
resultHandler.OnSuccess();
}
catch (Exception exc)
{
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
handler.SaveChanges();
saveHandler.SaveChanges();
log.Error("Failed to start new purchase: " + exc);
handler.OnFailure();
resultHandler.OnFailure();
}
}
@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore
throw new Exception("CID received from storage request was not protected.");
}
handler.SaveChanges();
saveHandler.SaveChanges();
Log("Saved new purchaseId: " + entry.PurchaseId);
return request;
}
@ -289,7 +299,7 @@ namespace AutoClient.Modes.FolderStore
Log("Request failed to start. State: " + update.State);
entry.EncodedCid = string.Empty;
entry.PurchaseId = string.Empty;
handler.SaveChanges();
saveHandler.SaveChanges();
return;
}
}
@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore
}
catch (Exception exc)
{
handler.OnFailure();
resultHandler.OnFailure();
Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}");
throw;
}

56
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs
View File

@ -11,14 +11,16 @@ namespace AutoClient.Modes.FolderStore
private readonly JsonFile<FolderStatus> statusFile;
private readonly FolderStatus status;
private readonly BalanceChecker balanceChecker;
private readonly SlowModeHandler slowModeHandler;
private int changeCounter = 0;
private int failureCount = 0;
private int saveFolderJsonCounter = 0;
public FolderSaver(App app, LoadBalancer loadBalancer)
{
this.app = app;
this.loadBalancer = loadBalancer;
balanceChecker = new BalanceChecker(app);
slowModeHandler = new SlowModeHandler(app);
statusFile = new JsonFile<FolderStatus>(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename));
status = statusFile.Load();
@ -26,10 +28,11 @@ namespace AutoClient.Modes.FolderStore
public void Run()
{
saveFolderJsonCounter = 0;
var folderFiles = Directory.GetFiles(app.Config.FolderToStore);
if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore);
var saveFolderJsonCounter = 0;
balanceChecker.Check();
foreach (var folderFile in folderFiles)
{
@ -41,35 +44,30 @@ namespace AutoClient.Modes.FolderStore
SaveFile(folderFile);
}
if (failureCount > 3)
{
app.Log.Error("Failure count reached threshold. Stopping...");
app.Cts.Cancel();
return;
}
if (changeCounter > 1)
{
changeCounter = 0;
saveFolderJsonCounter++;
if (saveFolderJsonCounter > 5)
{
saveFolderJsonCounter = 0;
if (failureCount > 0)
{
app.Log.Log($"Failure count is reset. (Was: {failureCount})");
failureCount = 0;
}
balanceChecker.Check();
SaveFolderSaverJsonFile();
}
}
slowModeHandler.Check();
CheckAndSaveChanges();
statusFile.Save(status);
Thread.Sleep(100);
}
}
private void CheckAndSaveChanges()
{
if (changeCounter > 1)
{
changeCounter = 0;
saveFolderJsonCounter++;
if (saveFolderJsonCounter > 5)
{
saveFolderJsonCounter = 0;
balanceChecker.Check();
SaveFolderSaverJsonFile();
}
}
}
private void SaveFile(string folderFile)
{
var localFilename = Path.GetFileName(folderFile);
@ -114,7 +112,6 @@ namespace AutoClient.Modes.FolderStore
}
private const int MinCodexStorageFilesize = 262144;
private readonly Random random = new Random();
private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " +
$"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " +
$"file is larger than the minimal size. The following is pseudo-random: ";
@ -135,7 +132,7 @@ namespace AutoClient.Modes.FolderStore
{
var fixedLength = entry.Filename.PadRight(35);
var prefix = $"[{fixedLength}] ";
return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this);
return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler);
}
public void SaveChanges()
@ -143,10 +140,5 @@ namespace AutoClient.Modes.FolderStore
statusFile.Save(status);
changeCounter++;
}
public void OnFailure()
{
failureCount++;
}
}
}

54
Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs Normal file
View File

@ -0,0 +1,54 @@
namespace AutoClient.Modes.FolderStore
{
public class SlowModeHandler : IFileSaverResultHandler
{
private readonly App app;
private int failureCount = 0;
private bool slowMode = false;
private int recoveryCount = 0;
public SlowModeHandler(App app)
{
this.app = app;
}
public void OnSuccess()
{
failureCount = 0;
if (slowMode)
{
recoveryCount++;
if (recoveryCount > 3)
{
Log("Recovery limit reached. Exiting slow mode.");
slowMode = false;
failureCount = 0;
}
}
}
public void OnFailure()
{
failureCount++;
if (failureCount > 3 && !slowMode)
{
Log("Failure limit reached. Entering slow mode.");
slowMode = true;
recoveryCount = 0;
}
}
public void Check()
{
if (slowMode)
{
Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes));
}
}
private void Log(string msg)
{
app.Log.Log(msg);
}
}
}

3
Tools/TestNetRewarder/Configuration.cs
View File

@ -33,6 +33,9 @@ namespace TestNetRewarder
[Uniform("proof-submitted-events", "pse", "PROOFSUBMITTEDEVENTS", false, "When greater than zero, chain event summary will include proof-submitted events.")]
public int ShowProofSubmittedEvents { get; set; } = 0; // Defaulted to zero, aprox 7 to 10 such events every 2 minutes in testnet (from autoclient alone!)
[Uniform("proof-period-report-hours", "pprh", "PROOFPERIODREPORTHOURS", false, "Frequency in hours with which proof period reports are created.")]
public int ProofReportHours { get; set; } = 24;
public string LogPath
{
get

4
Tools/TestNetRewarder/Processor.cs
View File

@ -22,6 +22,8 @@ namespace TestNetRewarder
this.log = log;
lastPeriodUpdateUtc = DateTime.UtcNow;
if (config.ProofReportHours < 1) throw new Exception("ProofReportHours must be one or greater");
builder = new RequestBuilder();
eventsFormatter = new EventsFormatter(config);
@ -79,7 +81,7 @@ namespace TestNetRewarder
private void ProcessPeriodUpdate()
{
if (config.ShowProofPeriodReports < 1) return;
if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(1.0))) return;
if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(config.ProofReportHours))) return;
lastPeriodUpdateUtc = DateTime.UtcNow;
eventsFormatter.ProcessPeriodReports(chainState.PeriodMonitor.GetAndClearReports());