From a676e0463d04e0b3b3f5c7daa117b04a7036769e Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Mar 2025 14:06:17 +0100 Subject: [PATCH] Sets up asserting of balances in case of failed contract. --- Framework/KubernetesWorkflow/LogHandler.cs | 4 + Framework/Utils/EthAddress.cs | 10 ++ ProjectPlugins/CodexClient/CodexAccess.cs | 4 +- ProjectPlugins/CodexClient/CodexClient.csproj | 1 + .../CodexClient/StoragePurchaseContract.cs | 23 +++- .../MarketTests/ContractFailedTest.cs | 108 ++++++++++++++---- .../MarketplaceAutoBootstrapDistTest.cs | 46 ++++++-- Tests/CodexReleaseTests/Parallelism.cs | 2 +- .../Utils/EthAddressEqualityTests.cs | 46 ++++++++ 9 files changed, 209 insertions(+), 35 deletions(-) create mode 100644 Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs diff --git a/Framework/KubernetesWorkflow/LogHandler.cs b/Framework/KubernetesWorkflow/LogHandler.cs index af77ef75..b8a8fa4b 100644 --- a/Framework/KubernetesWorkflow/LogHandler.cs +++ b/Framework/KubernetesWorkflow/LogHandler.cs @@ -40,6 +40,10 @@ namespace KubernetesWorkflow protected override void ProcessLine(string line) { + // This line is not useful and has no topic so we can't filter it with + // normal log-level controls. + if (line.Contains("Received JSON-RPC response") && !line.Contains("topics=")) return; + LogFile.WriteRaw(line); } } diff --git a/Framework/Utils/EthAddress.cs b/Framework/Utils/EthAddress.cs index 61c2776c..a16f4b46 100644 --- a/Framework/Utils/EthAddress.cs +++ b/Framework/Utils/EthAddress.cs @@ -30,5 +30,15 @@ { return Address; } + + public static bool operator ==(EthAddress a, EthAddress b) + { + return a.Address == b.Address; + } + + public static bool operator !=(EthAddress a, EthAddress b) + { + return a.Address != b.Address; + } } } diff --git a/ProjectPlugins/CodexClient/CodexAccess.cs b/ProjectPlugins/CodexClient/CodexAccess.cs index ca11ebee..e9a49224 100644 --- a/ProjectPlugins/CodexClient/CodexAccess.cs +++ b/ProjectPlugins/CodexClient/CodexAccess.cs @@ -11,7 +11,7 @@ namespace CodexClient private readonly ILog log; private readonly IHttpFactory httpFactory; private readonly IProcessControl processControl; - private ICodexInstance instance; + private readonly ICodexInstance instance; private readonly Mapper mapper = new Mapper(); public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance) @@ -25,8 +25,6 @@ namespace CodexClient public void Stop(bool waitTillStopped) { processControl.Stop(waitTillStopped); - // Prevents accidental use after stop: - instance = null!; } public IDownloadedLog DownloadLog(string additionalName = "") diff --git a/ProjectPlugins/CodexClient/CodexClient.csproj b/ProjectPlugins/CodexClient/CodexClient.csproj index cc417a9e..17a76836 100644 --- a/ProjectPlugins/CodexClient/CodexClient.csproj +++ b/ProjectPlugins/CodexClient/CodexClient.csproj @@ -30,6 +30,7 @@ + diff --git a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs index 618ca1c3..a1ae1394 100644 --- a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs +++ b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs @@ -1,4 +1,5 @@ using CodexClient.Hooks; +using CodexContractsPlugin.Marketplace; using Logging; using Newtonsoft.Json; using Utils; @@ -14,7 +15,7 @@ namespace CodexClient void WaitForStorageContractSubmitted(); void WaitForStorageContractStarted(); void WaitForStorageContractFinished(); - void WaitForContractFailed(); + void WaitForContractFailed(MarketplaceConfig config); } public class StoragePurchaseContract : IStoragePurchaseContract @@ -99,7 +100,7 @@ namespace CodexClient AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished)); } - public void WaitForContractFailed() + public void WaitForContractFailed(MarketplaceConfig config) { if (!contractStartedUtc.HasValue) { @@ -107,9 +108,27 @@ namespace CodexClient } var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value; var timeout = (Purchase.Duration - currentContractTime) + gracePeriod; + var minTimeout = TimeNeededToFailEnoughProofsToFreeASlot(config); + + if (timeout < minTimeout) + { + throw new ArgumentOutOfRangeException( + $"Test is misconfigured. Assuming a proof is required every period, it will take {Time.FormatDuration(minTimeout)} " + + $"to fail enough proofs for a slot to be freed. But, the storage contract will complete in {Time.FormatDuration(timeout)}. " + + $"Increase the duration." + ); + } + WaitForStorageContractState(timeout, "failed"); } + private TimeSpan TimeNeededToFailEnoughProofsToFreeASlot(MarketplaceConfig config) + { + var numMissedProofsRequiredForFree = config.Collateral.MaxNumberOfSlashes; + var timePerProof = TimeSpan.FromSeconds(config.Proofs.Period); + return timePerProof * (numMissedProofsRequiredForFree + 1); + } + private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000) { var waitStart = DateTime.UtcNow; diff --git a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs index 455306be..5c1d3be3 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs @@ -1,6 +1,9 @@ using CodexClient; +using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; +using CodexPlugin; using NUnit.Framework; +using System.Numerics; using Utils; namespace CodexReleaseTests.MarketTests @@ -21,7 +24,7 @@ namespace CodexReleaseTests.MarketTests { var hosts = StartHosts(); var client = StartClients().Single(); - StartValidator(); + var validator = StartValidator(); var request = CreateStorageRequest(client); @@ -32,32 +35,97 @@ namespace CodexReleaseTests.MarketTests AssertContractSlotsAreFilledByHosts(request, hosts); hosts.Stop(waitTillStopped: true); + + var config = GetContracts().Deployment.Config; + request.WaitForContractFailed(config); - WaitForSlotFreedEvents(); - AssertProofMissedReports(); + var frees = GetOnChainSlotFrees(hosts); + Assert.That(frees.Length, Is.EqualTo( + request.Purchase.MinRequiredNumberOfNodes - request.Purchase.NodeFailureTolerance)); - request.WaitForContractFailed(); + var periodReports = GetPeriodMonitorReports(); + var missedProofs = periodReports.Reports.SelectMany(r => r.MissedProofs).ToArray(); + AssertEnoughProofsWereMissedForSlotFree(frees, missedProofs, config); + + AssertClientPaidNothing(client); + AssertValidatorWasPaidPerMissedProof(validator, request, missedProofs, config); + AssertHostCollateralWasBurned(hosts, request); } - private void WaitForSlotFreedEvents() + private void AssertClientPaidNothing(ICodexNode client) { - Log(nameof(WaitForSlotFreedEvents)); + AssertTstBalance(client, StartingBalanceTST.Tst(), "Client should not have paid for failed contract."); + } - var start = DateTime.UtcNow; - var timeout = CalculateContractFailTimespan(); + private void AssertValidatorWasPaidPerMissedProof(ICodexNode validator, IStoragePurchaseContract request, PeriodProofMissed[] missedProofs, MarketplaceConfig config) + { + var rewardPerMissedProof = GetValidatorRewardPerMissedProof(request, config); + var totalValidatorReward = rewardPerMissedProof * missedProofs.Length; - while (DateTime.UtcNow < start + timeout) + AssertTstBalance(validator, StartingBalanceTST.Tst() + totalValidatorReward, $"Validator is rewarded per slot marked as missing. " + + $"numberOfMissedProofs: {missedProofs.Length} rewardPerMissedProof: {rewardPerMissedProof}"); + } + + private TestToken GetCollatoralPerSlot(IStoragePurchaseContract request) + { + var slotSize = new ByteSize(Convert.ToInt64(request.GetStatus()!.Request.Ask.SlotSize)); + return new TestToken(request.Purchase.CollateralPerByte.TstWei * slotSize.SizeInBytes); + } + + private void AssertHostCollateralWasBurned(ICodexNodeGroup hosts, IStoragePurchaseContract request) + { + var slotFills = GetOnChainSlotFills(hosts); + foreach (var host in hosts) { - var events = GetContracts().GetEvents(GetTestRunTimeRange()); - var slotFreed = events.GetSlotFreedEvents(); - if (slotFreed.Length == NumberOfSlots) - { - Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}"); - return; - } - GetContracts().WaitUntilNextPeriod(); + AssertHostCollateralWasBurned(host, slotFills, request); } - Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}"); + } + + private void AssertHostCollateralWasBurned(ICodexNode host, SlotFill[] slotFills, IStoragePurchaseContract request) + { + // In case of a failed contract, the entire slotColateral is lost. + var filledByHost = slotFills.Where(f => f.Host.EthAddress == host.EthAddress).ToArray(); + var numSlotsOfHost = filledByHost.Length; + var collatoralPerSlot = GetCollatoralPerSlot(request); + var totalCost = collatoralPerSlot * numSlotsOfHost; + + AssertTstBalance(host, StartingBalanceTST.Tst() - totalCost, $"Host has lost collateral for each slot. " + + $"numberOfSlotsByHost: {numSlotsOfHost} collateralPerSlot: {collatoralPerSlot}"); + } + + private TestToken GetValidatorRewardPerMissedProof(IStoragePurchaseContract request, MarketplaceConfig config) + { + var collatoralPerSlot = GetCollatoralPerSlot(request); + var slashPercentage = config.Collateral.SlashPercentage; + var validatorRewardPercentage = config.Collateral.ValidatorRewardPercentage; + + var rewardPerMissedProof = + PercentageOf( + PercentageOf(collatoralPerSlot, slashPercentage), + validatorRewardPercentage); + + return rewardPerMissedProof; + } + + private TestToken PercentageOf(TestToken value, byte percentage) + { + var p = new BigInteger(percentage); + return new TestToken((value.TstWei * p) / 100); + } + + private void AssertEnoughProofsWereMissedForSlotFree(SlotFree[] frees, PeriodProofMissed[] missedProofs, MarketplaceConfig config) + { + foreach (var free in frees) + { + AssertEnoughProofsWereMissedForSlotFree(free, missedProofs, config); + } + } + + private void AssertEnoughProofsWereMissedForSlotFree(SlotFree free, PeriodProofMissed[] missedProofs, MarketplaceConfig config) + { + var missedByHost = missedProofs.Where(p => p.Host != null && p.Host.Address == free.Host.EthAddress.Address).ToArray(); + var maxNumMissedProofsBeforeFreeSlot = config.Collateral.MaxNumberOfSlashes; + Assert.That(missedByHost.Length, Is.EqualTo(maxNumMissedProofsBeforeFreeSlot)); } private TimeSpan CalculateContractFailTimespan() @@ -92,8 +160,8 @@ namespace CodexReleaseTests.MarketTests var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB())); return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { - Duration = TimeSpan.FromHours(1.0), - Expiry = TimeSpan.FromHours(0.2), + Duration = TimeSpan.FromMinutes(10.0), + Expiry = TimeSpan.FromMinutes(5.0), MinRequiredNumberOfNodes = NumberOfSlots, NodeFailureTolerance = 1, PricePerBytePerSecond = pricePerBytePerSecond, diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index 17d83e3a..04957541 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -68,8 +68,8 @@ namespace CodexReleaseTests.MarketTests var hosts = StartCodex(NumberOfHosts, s => s .WithName("host") .WithBlockTTL(HostBlockTTL) - .WithBlockMaintenanceNumber(100000) - .WithBlockMaintenanceInterval(HostBlockTTL / 4) + .WithBlockMaintenanceNumber(100) + .WithBlockMaintenanceInterval(HostBlockTTL / 2) .EnableMarketplace(GetGeth(), GetContracts(), m => m .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst()) .AsStorageNode() @@ -236,6 +236,29 @@ namespace CodexReleaseTests.MarketTests }).ToArray(); } + public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts, string purchaseId) + { + var fills = GetOnChainSlotFrees(possibleHosts); + return fills.Where(f => f + .SlotFreedEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant()) + .ToArray(); + } + + public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts) + { + var events = GetContracts().GetEvents(GetTestRunTimeRange()); + var fills = GetOnChainSlotFills(possibleHosts); + var frees = events.GetSlotFreedEvents(); + return frees.Select(f => + { + var matchingFill = fills.Single(fill => fill.SlotFilledEvent.RequestId == f.RequestId && + fill.SlotFilledEvent.SlotIndex == f.SlotIndex); + + return new SlotFree(f, matchingFill.Host); + + }).ToArray(); + } + protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts) { var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts); @@ -359,13 +382,6 @@ namespace CodexReleaseTests.MarketTests }, description); } - protected void AssertEnoughProofMissedForSlotFree(ICodexNodeGroup hosts) - { - var slotFills = GetOnChainSlotFills(hosts); - - todo for each filled slot, there should be enough proofs missed to trigger the slot-free event. - } - public class SlotFill { public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host) @@ -378,6 +394,18 @@ namespace CodexReleaseTests.MarketTests public ICodexNode Host { get; } } + public class SlotFree + { + public SlotFree(SlotFreedEventDTO slotFreedEvent, ICodexNode host) + { + SlotFreedEvent = slotFreedEvent; + Host = host; + } + + public SlotFreedEventDTO SlotFreedEvent { get; } + public ICodexNode Host { get; } + } + private class MarketplaceHandle { public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor monitor) diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs index cd0a4299..a1b26c73 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/CodexReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -[assembly: LevelOfParallelism(10)] +[assembly: LevelOfParallelism(1)] namespace CodexReleaseTests.DataTests { } diff --git a/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs b/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs new file mode 100644 index 00000000..72cb8a67 --- /dev/null +++ b/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs @@ -0,0 +1,46 @@ +using GethPlugin; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Utils; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class EthAddressEqualityTests + { + [Test] + [Combinatorial] + public void Equal( + [Values(1, 2, 3, 4, 5)] int runs + ) + { + var account = EthAccountGenerator.GenerateNew(); + + var str = account.EthAddress.Address; + + var addr = new EthAddress(str); + + Assert.That(addr, Is.EqualTo(account.EthAddress)); + Assert.That(addr == account.EthAddress); + Assert.That(!(addr != account.EthAddress)); + } + + [Test] + [Combinatorial] + public void NotEqual( + [Values(1, 2, 3, 4, 5)] int runs + ) + { + var account1 = EthAccountGenerator.GenerateNew(); + var account2 = EthAccountGenerator.GenerateNew(); + + Assert.That(account1.EthAddress, Is.Not.EqualTo(account2.EthAddress)); + Assert.That(account1.EthAddress != account2.EthAddress); + Assert.That(!(account1.EthAddress == account2.EthAddress)); + } + } +}