logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-03 22:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
ThatBen 2025-03-11 15:00:27 +01:00
parent c47d133db0
commit cec27b2cf7
No known key found for this signature in database
GPG Key ID: 62C543548433D43E
19 changed files with 203 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ProjectPlugins/CodexClient/CodexAccess.cs
View File

@ -143,7 +143,7 @@ namespace CodexClient
return mapper.Map(OnCodex(api => api.ListDataAsync()));
}
public StorageAvailability SalesAvailability(StorageAvailability request)
public StorageAvailability SalesAvailability(CreateStorageAvailability request)
{
var body = mapper.Map(request);
var read = OnCodex(api => api.OfferStorageAsync(body));

12
ProjectPlugins/CodexClient/Mapper.cs
View File

@ -37,7 +37,7 @@ namespace CodexClient
};
}
public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
public CodexOpenApi.SalesAvailabilityCREATE Map(CreateStorageAvailability availability)
{
return new CodexOpenApi.SalesAvailabilityCREATE
{
@ -71,15 +71,13 @@ namespace CodexClient
{
return new StorageAvailability
(
availability.Id,
ToByteSize(availability.TotalSize),
ToTimespan(availability.Duration),
new TestToken(ToBigIng(availability.MinPricePerBytePerSecond)),
new TestToken(ToBigIng(availability.TotalCollateral))
)
{
Id = availability.Id,
FreeSpace = ToByteSize(availability.FreeSize),
};
new TestToken(ToBigIng(availability.TotalCollateral)),
ToByteSize(availability.FreeSize)
);
}
public StoragePurchase Map(CodexOpenApi.Purchase purchase)

6
ProjectPlugins/CodexClient/MarketplaceAccess.cs
View File

@ -6,7 +6,7 @@ namespace CodexClient
{
public interface IMarketplaceAccess
{
string MakeStorageAvailable(StorageAvailability availability);
string MakeStorageAvailable(CreateStorageAvailability availability);
StorageAvailability[] GetAvailabilities();
IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
}
@ -49,7 +49,7 @@ namespace CodexClient
return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
}
public string MakeStorageAvailable(StorageAvailability availability)
public string MakeStorageAvailable(CreateStorageAvailability availability)
{
availability.Log(log);
@ -77,7 +77,7 @@ namespace CodexClient
public class MarketplaceUnavailable : IMarketplaceAccess
{
public string MakeStorageAvailable(StorageAvailability availability)
public string MakeStorageAvailable(CreateStorageAvailability availability)
{
Unavailable();
throw new NotImplementedException();

43
ProjectPlugins/CodexClient/MarketplaceTypes.cs
View File

@ -84,9 +84,9 @@ namespace CodexClient
//public PoRParameters Por { get; set; }
}
public class StorageAvailability
public class CreateStorageAvailability
{
public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
public CreateStorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
{
TotalSpace = totalSpace;
MaxDuration = maxDuration;
@ -94,20 +94,49 @@ namespace CodexClient
TotalCollateral = totalCollateral;
}
public string Id { get; set; } = string.Empty;
public ByteSize TotalSpace { get; }
public TimeSpan MaxDuration { get; }
public TestToken MinPricePerBytePerSecond { get; }
public TestToken TotalCollateral { get; }
public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
public TestToken TotalCollateral { get; }
public void Log(ILog log)
{
log.Log($"Storage Availability: (" +
log.Log($"Create storage Availability: (" +
$"totalSize: {TotalSpace}, " +
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
$"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
$"totalCollateral: {TotalCollateral})");
}
}
public class StorageAvailability
{
public StorageAvailability(string id, ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral, ByteSize freeSpace)
{
Id = id;
TotalSpace = totalSpace;
MaxDuration = maxDuration;
MinPricePerBytePerSecond = minPricePerBytePerSecond;
TotalCollateral = totalCollateral;
FreeSpace = freeSpace;
}
public string Id { get; }
public ByteSize TotalSpace { get; }
public TimeSpan MaxDuration { get; }
public TestToken MinPricePerBytePerSecond { get; }
public TestToken TotalCollateral { get; }
public ByteSize FreeSpace { get; }
public void Log(ILog log)
{
log.Log($"Storage Availability: (" +
$"id: {Id}, " +
$"totalSize: {TotalSpace}, " +
$"maxDuration: {Time.FormatDuration(MaxDuration)}, " +
$"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
$"totalCollateral: {TotalCollateral}, " +
$"freeSpace: {FreeSpace})");
}
}
}

1
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs
View File

@ -1,5 +1,4 @@
using CodexContractsPlugin.Marketplace;
using System.Collections.Generic;
using Utils;
namespace CodexContractsPlugin.ChainMonitor

3
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -1,6 +1,5 @@
using BlockchainUtils;
using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using System.Numerics;
using Utils;
@ -79,7 +78,7 @@ namespace CodexContractsPlugin.ChainMonitor
throw new Exception(msg);
}
log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
log.Debug($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
// Run through each block and apply the events to the state in order.
var span = events.BlockInterval.TimeRange.Duration;

22
ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs
View File

@ -39,8 +39,6 @@ namespace CodexContractsPlugin.ChainMonitor
private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests)
{
log.Log("Creating report for period " + periodNumber);
ulong total = 0;
ulong required = 0;
var missed = new List<PeriodProofMissed>();
@ -63,7 +61,9 @@ namespace CodexContractsPlugin.ChainMonitor
}
}
}
reports.Add(new PeriodReport(periodNumber, total, required, missed.ToArray()));
var report = new PeriodReport(periodNumber, total, required, missed.ToArray());
log.Log($"Period report: {report}");
reports.Add(report);
}
}
@ -108,6 +108,16 @@ namespace CodexContractsPlugin.ChainMonitor
public ulong TotalNumSlots { get; }
public ulong TotalProofsRequired { get; }
public PeriodProofMissed[] MissedProofs { get; }
public override string ToString()
{
var missed = "None";
if (MissedProofs.Length > 0)
{
missed = string.Join("+", MissedProofs.Select(p => $"{p.FormatHost()} missed {p.Request.Request.Id} slot {p.SlotIndex}"));
}
return $"Period:{PeriodNumber}=[Slots:{TotalNumSlots},ProofsRequired:{TotalProofsRequired},ProofsMissed:{missed}]";
}
}
public class PeriodProofMissed
@ -122,5 +132,11 @@ namespace CodexContractsPlugin.ChainMonitor
public EthAddress? Host { get; }
public IChainStateRequest Request { get; }
public int SlotIndex { get; }
public string FormatHost()
{
if (Host == null) return "Unknown host";
return Host.Address;
}
}
}

3
ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs
View File

@ -3,11 +3,8 @@ using CodexContractsPlugin.Marketplace;
using GethPlugin;
using Logging;
using Nethereum.ABI;
using Nethereum.ABI.FunctionEncoding.Attributes;
using Nethereum.Contracts;
using Nethereum.Hex.HexConvertors.Extensions;
using Nethereum.Util;
using NethereumWorkflow;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Utils;

49
Tests/CodexReleaseTests/MarketTests/ChainMonitor.cs Normal file
View File

@ -0,0 +1,49 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using Logging;
namespace CodexReleaseTests.MarketTests
{
public class ChainMonitor
{
private readonly ChainState chainMonitor;
private readonly TimeSpan interval;
private CancellationTokenSource cts = new CancellationTokenSource();
private Task worker = null!;
public ChainMonitor(ILog log, ICodexContracts contracts, DateTime startUtc, TimeSpan interval)
{
chainMonitor = new ChainState(log, contracts, new DoNothingChainEventHandler(), startUtc, true);
this.interval = interval;
}
public void Start()
{
cts = new CancellationTokenSource();
worker = Task.Run(Worker);
}
public void Stop()
{
cts.Cancel();
worker.Wait();
worker = null!;
cts = null!;
}
public PeriodMonitorResult GetPeriodReports()
{
return chainMonitor.PeriodMonitor.GetAndClearReports();
}
private void Worker()
{
while (!cts.IsCancellationRequested)
{
Thread.Sleep(interval);
chainMonitor.Update();
}
}
}
}

21
Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs
View File

@ -7,14 +7,16 @@ namespace CodexReleaseTests.MarketTests
{
public class ContractFailedTest : MarketplaceAutoBootstrapDistTest
{
protected override int NumberOfHosts => 4;
private const int FilesizeMb = 10;
private const int NumberOfSlots = 3;
protected override int NumberOfHosts => 6;
protected override int NumberOfClients => 1;
protected override ByteSize HostAvailabilitySize => 1.GB();
protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(1.0);
protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
private readonly TestToken pricePerBytePerSecond = 10.TstWei();
[Test]
[Ignore("Disabled for now: Test is unstable.")]
public void ContractFailed()
{
var hosts = StartHosts();
@ -32,6 +34,7 @@ namespace CodexReleaseTests.MarketTests
hosts.Stop(waitTillStopped: true);
WaitForSlotFreedEvents();
AssertProofMissedReports();
request.WaitForContractFailed();
}
@ -47,7 +50,7 @@ namespace CodexReleaseTests.MarketTests
{
var events = GetContracts().GetEvents(GetTestRunTimeRange());
var slotFreed = events.GetSlotFreedEvents();
if (slotFreed.Length == NumberOfHosts)
if (slotFreed.Length == NumberOfSlots)
{
Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
return;
@ -86,16 +89,16 @@ namespace CodexReleaseTests.MarketTests
private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
{
var cid = client.UploadFile(GenerateTestFile(5.MB()));
var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
Duration = TimeSpan.FromHours(1.0),
Expiry = TimeSpan.FromHours(0.2),
MinRequiredNumberOfNodes = (uint)NumberOfHosts,
NodeFailureTolerance = (uint)(NumberOfHosts / 2),
MinRequiredNumberOfNodes = NumberOfSlots,
NodeFailureTolerance = 1,
PricePerBytePerSecond = pricePerBytePerSecond,
ProofProbability = 1, // Require a proof every period
CollateralPerByte = 1.Tst()
CollateralPerByte = 1.TstWei()
});
}
}

3
Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs
View File

@ -20,6 +20,7 @@ namespace CodexReleaseTests.MarketTests
{
var hosts = StartHosts();
var client = StartClients().Single();
AssertHostAvailabilitiesAreEmpty(hosts);
var request = CreateStorageRequest(client);
@ -34,12 +35,12 @@ namespace CodexReleaseTests.MarketTests
AssertClientHasPaidForContract(pricePerBytePerSecond, client, request, hosts);
AssertHostsWerePaidForContract(pricePerBytePerSecond, request, hosts);
AssertHostsCollateralsAreUnchanged(hosts);
AssertHostAvailabilitiesAreEmpty(hosts);
}
private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
{
var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
var config = GetContracts().Deployment.Config;
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
Duration = GetContractDuration(),

75
Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
View File

@ -1,5 +1,6 @@
using CodexClient;
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
using CodexPlugin;
using CodexTests;
@ -22,13 +23,17 @@ namespace CodexReleaseTests.MarketTests
base.LifecycleStart(lifecycle);
var geth = StartGethNode(s => s.IsMiner());
var contracts = Ci.StartCodexContracts(geth);
handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
var monitor = new ChainMonitor(lifecycle.Log, contracts, lifecycle.TestStart, TimeSpan.FromSeconds(1.0));
monitor.Start();
handles.Add(lifecycle, new MarketplaceHandle(geth, contracts, monitor));
}
protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
{
base.LifecycleStop(lifecycle, result);
handles[lifecycle].Monitor.Stop();
handles.Remove(lifecycle);
base.LifecycleStop(lifecycle, result);
}
protected IGethNode GetGeth()
@ -44,18 +49,27 @@ namespace CodexReleaseTests.MarketTests
protected TimeSpan GetPeriodDuration()
{
var config = GetContracts().Deployment.Config;
return TimeSpan.FromSeconds(((double)config.Proofs.Period));
return TimeSpan.FromSeconds(config.Proofs.Period);
}
protected PeriodMonitorResult GetPeriodMonitorReports()
{
return handles[Get()].Monitor.GetPeriodReports();
}
protected abstract int NumberOfHosts { get; }
protected abstract int NumberOfClients { get; }
protected abstract ByteSize HostAvailabilitySize { get; }
protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0);
public ICodexNodeGroup StartHosts()
{
var hosts = StartCodex(NumberOfHosts, s => s
.WithName("host")
.WithBlockTTL(HostBlockTTL)
.WithBlockMaintenanceNumber(100000)
.WithBlockMaintenanceInterval(HostBlockTTL / 4)
.EnableMarketplace(GetGeth(), GetContracts(), m => m
.WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
.AsStorageNode()
@ -67,17 +81,49 @@ namespace CodexReleaseTests.MarketTests
{
AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartHosts));
AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartHosts));
host.Marketplace.MakeStorageAvailable(new StorageAvailability(
var spaceBefore = host.Space();
Assert.That(spaceBefore.QuotaReservedBytes, Is.EqualTo(0));
Assert.That(spaceBefore.QuotaUsedBytes, Is.EqualTo(0));
host.Marketplace.MakeStorageAvailable(new CreateStorageAvailability(
totalSpace: HostAvailabilitySize,
maxDuration: HostAvailabilityMaxDuration,
minPricePerBytePerSecond: 1.TstWei(),
totalCollateral: 999999.Tst())
);
var spaceAfter = host.Space();
Assert.That(spaceAfter.QuotaReservedBytes, Is.EqualTo(HostAvailabilitySize.SizeInBytes));
Assert.That(spaceAfter.QuotaUsedBytes, Is.EqualTo(0));
}
return hosts;
}
public void AssertHostAvailabilitiesAreEmpty(IEnumerable<ICodexNode> hosts)
{
var retry = GetAvailabilitySpaceAssertRetry();
retry.Run(() =>
{
foreach (var host in hosts)
{
AssertHostAvailabilitiesAreEmpty(host);
}
});
}
private void AssertHostAvailabilitiesAreEmpty(ICodexNode host)
{
var availabilities = host.Marketplace.GetAvailabilities();
foreach (var a in availabilities)
{
if (a.FreeSpace.SizeInBytes != a.TotalSpace.SizeInBytes)
{
throw new Exception(nameof(AssertHostAvailabilitiesAreEmpty) + $" free: {a.FreeSpace} total: {a.TotalSpace}");
}
}
}
public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message)
{
AssertTstBalance(node.EthAddress, expectedBalance, message);
@ -121,6 +167,14 @@ namespace CodexReleaseTests.MarketTests
onFail: f => { });
}
private Retry GetAvailabilitySpaceAssertRetry()
{
return new Retry("AssertAvailabilitySpace",
maxTimeout: HostBlockTTL * 3,
sleepAfterFail: TimeSpan.FromSeconds(10.0),
onFail: f => { });
}
private TestToken GetTstBalance(ICodexNode node)
{
return GetContracts().GetTestTokenBalance(node);
@ -305,6 +359,13 @@ namespace CodexReleaseTests.MarketTests
}, description);
}
protected void AssertEnoughProofMissedForSlotFree(ICodexNodeGroup hosts)
{
var slotFills = GetOnChainSlotFills(hosts);
todo for each filled slot, there should be enough proofs missed to trigger the slot-free event.
}
public class SlotFill
{
public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
@ -319,14 +380,16 @@ namespace CodexReleaseTests.MarketTests
private class MarketplaceHandle
{
public MarketplaceHandle(IGethNode geth, ICodexContracts contracts)
public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor monitor)
{
Geth = geth;
Contracts = contracts;
Monitor = monitor;
}
public IGethNode Geth { get; }
public ICodexContracts Contracts { get; }
public ChainMonitor Monitor { get; }
}
}
}

2
Tests/CodexReleaseTests/Parallelism.cs
View File

@ -1,6 +1,6 @@
using NUnit.Framework;
[assembly: LevelOfParallelism(1)]
[assembly: LevelOfParallelism(10)]
namespace CodexReleaseTests.DataTests
{
}

1
Tests/DistTestCore/Helpers/AssertHelpers.cs
View File

@ -10,7 +10,6 @@ namespace DistTestCore.Helpers
{
try
{
Time.WaitUntil(() => {
var c = constraint.Resolve();
return c.ApplyTo(actual()).IsSuccess;

2
Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs
View File

@ -50,7 +50,7 @@ namespace ExperimentalTests.BasicTests
{
AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
var availability = new StorageAvailability(
var availability = new CreateStorageAvailability(
totalSpace: 10.GB(),
maxDuration: TimeSpan.FromMinutes(30),
minPricePerBytePerSecond: 1.TstWei(),

1
Tests/ExperimentalTests/CodexDistTest.cs
View File

@ -182,6 +182,7 @@ namespace CodexTests
public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
{
Assert.Fail("Depricated, use MarketplaceAutobootstrapDistTest assertBalances instead.");
AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
}

2
Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs
View File

@ -198,7 +198,7 @@ namespace ExperimentalTests.UtilityTests
.AsStorageNode()
.AsValidator()));
var availability = new StorageAvailability(
var availability = new CreateStorageAvailability(
totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()),
maxDuration: TimeSpan.FromMinutes(30),
minPricePerBytePerSecond: 1.TstWei(),

2
Tools/CodexNetDeployer/CodexNodeStarter.cs
View File

@ -69,7 +69,7 @@ namespace CodexNetDeployer
if (config.ShouldMakeStorageAvailable)
{
var availability = new StorageAvailability(
var availability = new CreateStorageAvailability(
totalSpace: config.StorageSell!.Value.MB(),
maxDuration: TimeSpan.FromSeconds(config.MaxDuration),
minPricePerBytePerSecond: config.MinPricePerBytePerSecond.TstWei(),

8
Tools/TestNetRewarder/EventsFormatter.cs
View File

@ -165,13 +165,7 @@ namespace TestNetRewarder
private void DescribeMissedProof(List<string> lines, PeriodProofMissed missedProof)
{
lines.Add($"[{FormatHost(missedProof.Host)}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})");
}
private string FormatHost(EthAddress? host)
{
if (host == null) return "Unknown host";
return host.Address;
lines.Add($"[{missedProof.FormatHost()}] missed proof for {FormatRequestId(missedProof.Request)} (slotIndex: {missedProof.SlotIndex})");
}
private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)