From 9c1a0b6942ef8f292ecbc94c9037fe48ce3c6f93 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 29 May 2025 16:37:20 +0200 Subject: [PATCH] Sets up repair test --- Framework/Utils/EthAccount.cs | 27 ++- Framework/Utils/EthAddress.cs | 20 +- .../Marketplace/Customizations.cs | 6 + .../MarketTests/ContractFailedTest.cs | 28 --- .../MarketTests/ContractRepairedTest.cs | 18 -- .../MarketplaceAutoBootstrapDistTest.cs | 54 ++++- .../MarketTests/RepairTest.cs | 185 ++++++++++++++++++ .../Utils/EthAccountEqualityTests.cs | 31 +++ 8 files changed, 317 insertions(+), 52 deletions(-) delete mode 100644 Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs create mode 100644 Tests/CodexReleaseTests/MarketTests/RepairTest.cs create mode 100644 Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs diff --git a/Framework/Utils/EthAccount.cs b/Framework/Utils/EthAccount.cs index 0898a30a..6119eb4e 100644 --- a/Framework/Utils/EthAccount.cs +++ b/Framework/Utils/EthAccount.cs @@ -1,7 +1,7 @@ namespace Utils { [Serializable] - public class EthAccount + public class EthAccount : IComparable { public EthAccount(EthAddress ethAddress, string privateKey) { @@ -12,9 +12,34 @@ public EthAddress EthAddress { get; } public string PrivateKey { get; } + public int CompareTo(EthAccount? other) + { + return PrivateKey.CompareTo(other!.PrivateKey); + } + + public override bool Equals(object? obj) + { + return obj is EthAccount token && PrivateKey == token.PrivateKey; + } + + public override int GetHashCode() + { + return HashCode.Combine(PrivateKey); + } + public override string ToString() { return EthAddress.ToString(); } + + public static bool operator ==(EthAccount a, EthAccount b) + { + return a.PrivateKey == b.PrivateKey; + } + + public static bool operator !=(EthAccount a, EthAccount b) + { + return a.PrivateKey != b.PrivateKey; + } } } diff --git a/Framework/Utils/EthAddress.cs b/Framework/Utils/EthAddress.cs index 61c2776c..7ac80093 100644 --- a/Framework/Utils/EthAddress.cs +++ b/Framework/Utils/EthAddress.cs @@ -6,7 +6,7 @@ } [Serializable] - public class EthAddress + public class EthAddress : IComparable { public EthAddress(string address) { @@ -15,10 +15,14 @@ public string Address { get; } + public int CompareTo(EthAddress? other) + { + return Address.CompareTo(other!.Address); + } + public override bool Equals(object? obj) { - return obj is EthAddress address && - Address == address.Address; + return obj is EthAddress token && Address == token.Address; } public override int GetHashCode() @@ -30,5 +34,15 @@ { return Address; } + + public static bool operator ==(EthAddress a, EthAddress b) + { + return a.Address == b.Address; + } + + public static bool operator !=(EthAddress a, EthAddress b) + { + return a.Address != b.Address; + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index 222dc631..c69dab8b 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -1,5 +1,6 @@ #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. using BlockchainUtils; +using Nethereum.Hex.HexConvertors.Extensions; using Newtonsoft.Json; using Utils; @@ -61,6 +62,11 @@ namespace CodexContractsPlugin.Marketplace [JsonIgnore] public BlockTimeEntry Block { get; set; } public EthAddress Host { get; set; } + + public override string ToString() + { + return $"SlotFilled:[host:{Host} request:{RequestId.ToHex()} slotIndex:{SlotIndex}]"; + } } public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex diff --git a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs index caeaed9b..d340ba26 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs @@ -1,5 +1,4 @@ using CodexClient; -using CodexContractsPlugin.Marketplace; using NUnit.Framework; using Utils; @@ -57,33 +56,6 @@ namespace CodexReleaseTests.MarketTests Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}"); } - private TimeSpan CalculateContractFailTimespan() - { - var config = GetContracts().Deployment.Config; - var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes); - var periodDuration = GetPeriodDuration(); - - // Each host could miss 1 proof per period, - // so the time we should wait is period time * requiredNum of missed proofs. - // Except: the proof requirement has a concept of "downtime": - // a segment of time where proof is not required. - // We calculate the probability of downtime and extend the waiting - // timeframe by a factor, such that all hosts are highly likely to have - // failed a sufficient number of proofs. - - float n = requiredNumMissedProofs; - return periodDuration * n * GetDowntimeFactor(config); - } - - private float GetDowntimeFactor(MarketplaceConfig config) - { - byte numBlocksInDowntimeSegment = config.Proofs.Downtime; - float downtime = numBlocksInDowntimeSegment; - float window = 256.0f; - var chanceOfDowntime = downtime / window; - return 1.0f + chanceOfDowntime + chanceOfDowntime; - } - private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) { var cid = client.UploadFile(GenerateTestFile(5.MB())); diff --git a/Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs deleted file mode 100644 index 752afe18..00000000 --- a/Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs +++ /dev/null @@ -1,18 +0,0 @@ -using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace CodexReleaseTests.MarketTests -{ - public class ContractRepairedTest - { - [Test] - [Ignore("TODO - Test in which a host fails, but the slot is repaired")] - public void ContractRepaired() - { - } - } -} diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index c14222df..74a1b6d5 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -80,6 +80,29 @@ namespace CodexReleaseTests.MarketTests return hosts; } + public ICodexNode StartOneHost() + { + var host = StartCodex(s => s + .WithName("singlehost") + .EnableMarketplace(GetGeth(), GetContracts(), m => m + .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst()) + .AsStorageNode() + ) + ); + + var config = GetContracts().Deployment.Config; + AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartOneHost)); + AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartOneHost)); + + host.Marketplace.MakeStorageAvailable(new StorageAvailability( + totalSpace: HostAvailabilitySize, + maxDuration: HostAvailabilityMaxDuration, + minPricePerBytePerSecond: 1.TstWei(), + totalCollateral: 999999.Tst()) + ); + return host; + } + public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message) { AssertTstBalance(node.EthAddress, expectedBalance, message); @@ -185,7 +208,7 @@ namespace CodexReleaseTests.MarketTests ); } - public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId) + public SlotFill[] GetOnChainSlotFills(IEnumerable possibleHosts, string purchaseId) { var fills = GetOnChainSlotFills(possibleHosts); return fills.Where(f => f @@ -193,7 +216,7 @@ namespace CodexReleaseTests.MarketTests .ToArray(); } - public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts) + public SlotFill[] GetOnChainSlotFills(IEnumerable possibleHosts) { var events = GetContracts().GetEvents(GetTestRunTimeRange()); var fills = events.GetSlotFilledEvents(); @@ -356,6 +379,33 @@ namespace CodexReleaseTests.MarketTests }, description); } + protected TimeSpan CalculateContractFailTimespan() + { + var config = GetContracts().Deployment.Config; + var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes); + var periodDuration = GetPeriodDuration(); + var gracePeriod = periodDuration; + + // Each host could miss 1 proof per period, + // so the time we should wait is period time * requiredNum of missed proofs. + // Except: the proof requirement has a concept of "downtime": + // a segment of time where proof is not required. + // We calculate the probability of downtime and extend the waiting + // timeframe by a factor, such that all hosts are highly likely to have + // failed a sufficient number of proofs. + + float n = requiredNumMissedProofs; + return gracePeriod + (periodDuration * n * GetDowntimeFactor(config)); + } + + private float GetDowntimeFactor(MarketplaceConfig config) + { + byte numBlocksInDowntimeSegment = config.Proofs.Downtime; + float downtime = numBlocksInDowntimeSegment; + float window = 256.0f; + var chanceOfDowntime = downtime / window; + return 1.0f + chanceOfDowntime + chanceOfDowntime; + } public class SlotFill { public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host) diff --git a/Tests/CodexReleaseTests/MarketTests/RepairTest.cs b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs new file mode 100644 index 00000000..6ae53a32 --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/RepairTest.cs @@ -0,0 +1,185 @@ +using CodexClient; +using Nethereum.Hex.HexConvertors.Extensions; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class RepairTest : MarketplaceAutoBootstrapDistTest + { + #region Setup + + private readonly ByteSize Filesize; + private readonly uint Slots; + private readonly uint Tolerance; + private readonly ByteSize EncodedFilesize; + private readonly ByteSize SlotSize; + + public RepairTest() + { + Filesize = 32.MB(); + Slots = 4; + Tolerance = 2; + + EncodedFilesize = new ByteSize(Filesize.SizeInBytes * (Slots / Tolerance)); + SlotSize = new ByteSize(EncodedFilesize.SizeInBytes / Slots); + Assert.That(IsPowerOfTwo(SlotSize)); + Assert.That(Slots, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 5; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => GetPeriodDuration() * 100; + + private static bool IsPowerOfTwo(ByteSize size) + { + var x = size.SizeInBytes; + return (x != 0) && ((x & (x - 1)) == 0); + } + + #endregion + + [Test] + [Combinatorial] + public void RollingRepairSingleFailure( + [Values(10)] int numFailures) + { + var hosts = StartHosts().ToList(); + var client = StartClients().Single(); + + var contract = CreateStorageRequest(client); + contract.WaitForStorageContractStarted(); + // All slots are filled. + + for (var i = 0; i < numFailures; i++) + { + Log($"Failure step: {i}"); + + // Start a new host. Add it to the back of the list: + hosts.Add(StartOneHost()); + + var fill = GetSlotFillByOldestHost(hosts); + + Log($"Causing failure for host: {fill.Host.GetName()} slotIndex: {fill.SlotFilledEvent.SlotIndex}"); + hosts.Remove(fill.Host); + fill.Host.Stop(waitTillStopped: true); + + // The slot should become free. + WaitForSlotFreedEvent(contract, fill.SlotFilledEvent.SlotIndex); + + // One of the other hosts should pick up the free slot. + WaitForNewSlotFilledEvent(contract, fill.SlotFilledEvent.SlotIndex); + } + } + + private void WaitForSlotFreedEvent(IStoragePurchaseContract contract, ulong slotIndex) + { + Log(nameof(WaitForSlotFreedEvent)); + var start = DateTime.UtcNow; + var timeout = CalculateContractFailTimespan(); + + while (DateTime.UtcNow < start + timeout) + { + var events = GetContracts().GetEvents(GetTestRunTimeRange()); + var slotsFreed = events.GetSlotFreedEvents(); + Log($"Slots freed this period: {slotsFreed.Length}"); + + foreach (var free in slotsFreed) + { + if (free.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant()) + { + if (free.SlotIndex == slotIndex) + { + Log("Found the correct slotFree event"); + return; + } + } + } + + GetContracts().WaitUntilNextPeriod(); + } + Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}"); + } + + private void WaitForNewSlotFilledEvent(IStoragePurchaseContract contract, ulong slotIndex) + { + Log(nameof(WaitForNewSlotFilledEvent)); + var start = DateTime.UtcNow; + var timeout = contract.Purchase.Expiry; + + while (DateTime.UtcNow < start + timeout) + { + var newTimeRange = new TimeRange(start, DateTime.UtcNow); // We only want to see new fill events. + var events = GetContracts().GetEvents(newTimeRange); + var slotFillEvents = events.GetSlotFilledEvents(); + + var matches = slotFillEvents.Where(f => + { + return + f.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant() && + f.SlotIndex == slotIndex; + }).ToArray(); + + if (matches.Length > 1) + { + var msg = string.Join(",", matches.Select(f => f.ToString())); + Assert.Fail($"Somehow, the slot got filled multiple times: {msg}"); + } + if (matches.Length == 1) + { + Log($"Found the correct new slotFilled event: {matches[0].ToString()}"); + } + + Thread.Sleep(TimeSpan.FromSeconds(15)); + } + Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}"); + } + + private SlotFill GetSlotFillByOldestHost(List hosts) + { + var fills = GetOnChainSlotFills(hosts); + var copy = hosts.ToArray(); + foreach (var host in copy) + { + var fill = GetFillByHost(host, fills); + if (fill == null) + { + // This host didn't fill anything. + // Move this one to the back of the list. + hosts.Remove(host); + hosts.Add(host); + } + else + { + return fill; + } + } + throw new Exception("None of the hosts seem to have filled a slot."); + } + + private SlotFill? GetFillByHost(ICodexNode host, SlotFill[] fills) + { + // If these is more than 1 fill by this host, the test is misconfigured. + // The availability size of the host should guarantee it can fill 1 slot maximum. + return fills.SingleOrDefault(f => f.Host.EthAddress == host.EthAddress); + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client) + { + var cid = client.UploadFile(GenerateTestFile(Filesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = TimeSpan.FromDays(2.0), + Expiry = TimeSpan.FromMinutes(10.0), + MinRequiredNumberOfNodes = Slots, + NodeFailureTolerance = Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 20, + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs b/Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs new file mode 100644 index 00000000..337516ae --- /dev/null +++ b/Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs @@ -0,0 +1,31 @@ +using GethPlugin; +using NUnit.Framework; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class EthAccountEqualityTests + { + [Test] + public void Accounts() + { + var account1 = EthAccountGenerator.GenerateNew(); + var account2 = EthAccountGenerator.GenerateNew(); + + Assert.That(account1, Is.EqualTo(account1)); + Assert.That(account1 == account1); + Assert.That(account1 != account2); + } + + [Test] + public void Addresses() + { + var address1 = EthAccountGenerator.GenerateNew().EthAddress; + var address2 = EthAccountGenerator.GenerateNew().EthAddress; + + Assert.That(address1, Is.EqualTo(address1)); + Assert.That(address1 == address1); + Assert.That(address1 != address2); + } + } +}