diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs index 177fe2e9..fd080820 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs @@ -19,9 +19,10 @@ namespace CodexContractsPlugin.ChainMonitor public void Log(ILog log) { log.Log($"Period report: {Period}"); + log.Log($" - Proofs required: {Required.Length}"); foreach (var r in Required) { - log.Log($" Required: {r.Describe()}"); + log.Log($" - {r.Describe()}"); } log.Log($" - Calls: {FunctionCalls.Length}"); foreach (var f in FunctionCalls) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs index 7332ed1c..5b560081 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs @@ -20,7 +20,7 @@ namespace CodexContractsPlugin.ChainMonitor public string Describe() { - return $"{Request.RequestId.ToHex()} slotIndex:{SlotIndex} by {Host}"; + return $"{Request.RequestId.ToHex()} slotId:{SlotId.ToHex()} slotIndex:{SlotIndex} by {Host}"; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs new file mode 100644 index 00000000..698b4afa --- /dev/null +++ b/Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs @@ -0,0 +1,95 @@ +using CodexClient; +using CodexContractsPlugin.ChainMonitor; +using CodexReleaseTests.Utils; +using Nethereum.Hex.HexConvertors.Extensions; +using NUnit.Framework; +using Utils; + +namespace CodexReleaseTests.MarketTests +{ + [TestFixture] + public class IsProofRequiredTest : MarketplaceAutoBootstrapDistTest + { + #region Setup + + private readonly PurchaseParams purchaseParams = new PurchaseParams( + nodes: 4, + tolerance: 2, + uploadFilesize: 32.MB() + ); + + public IsProofRequiredTest() + { + Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts)); + } + + protected override int NumberOfHosts => 6; + protected override int NumberOfClients => 1; + protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot. + protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0); + + #endregion + + [Test] + public void IsProofRequired() + { + var mins = TimeSpan.FromMinutes(10.0); + + StartHosts(); + StartValidator(); + var client = StartClients().Single(); + var purchase = CreateStorageRequest(client, mins); + purchase.WaitForStorageContractStarted(); + + var requestId = purchase.PurchaseId.HexToByteArray(); + var numSlots = purchaseParams.Nodes; + //var map = new Dictionary>(); + + Log($"Checking IsProofRequired every second for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(1)); + var requiredSlotIndices = new List(); + for (var i = 0; i < numSlots; i++) + { + if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i); + } + + var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow); + var blockNumber = GetGeth().GetSyncedBlockNumber(); + Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" + + $"{periodNumber.ToString().PadLeft(12, '0')} => " + + $"{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + + //var num = currentPeriod.PeriodNumber; + //if (!map.ContainsKey(num)) + //{ + // map.Add(num, requiredSlotIndices); + // Log($"Period {num} = required proof for slot indices {string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}"); + //} + //else + //{ + // var a = map[num]; + // CollectionAssert.AreEquivalent(a, requiredSlotIndices); + //} + } + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) + { + var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); + var config = GetContracts().Deployment.Config; + return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + Duration = minutes * 1.1, + Expiry = TimeSpan.FromMinutes(8.0), + MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, + NodeFailureTolerance = (uint)purchaseParams.Tolerance, + PricePerBytePerSecond = 10.TstWei(), + ProofProbability = 1, // One proof every period. Free slot as quickly as possible. + CollateralPerByte = 1.TstWei() + }); + } + } +} diff --git a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs index 9b7b08b8..02f7a84d 100644 --- a/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/StabilityTest.cs @@ -10,7 +10,7 @@ using Utils; namespace CodexReleaseTests.MarketTests { [TestFixture] - public class StabilityTest : MarketplaceAutoBootstrapDistTest, IPeriodMonitorEventHandler + public class StabilityTest : MarketplaceAutoBootstrapDistTest { #region Setup @@ -32,36 +32,63 @@ namespace CodexReleaseTests.MarketTests #endregion + private int numPeriods = 0; + private bool proofWasMissed = false; + [Test] [Combinatorial] public void Stability( [Values(10, 120)] int minutes) { - Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(TimeSpan.FromMinutes(minutes * 1.1))); + var mins = TimeSpan.FromMinutes(minutes); + var periodDuration = GetContracts().Deployment.Config.PeriodDuration; + Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(mins * 1.1)); - GetChainMonitor().PeriodMonitorEventHandler = this; + numPeriods = 0; + proofWasMissed = false; StartHosts(); StartValidator(); var client = StartClients().Single(); - var purchase = CreateStorageRequest(client, minutes); + var purchase = CreateStorageRequest(client, mins); purchase.WaitForStorageContractStarted(); - Log($"Contract should remain stable for {minutes} minutes."); - Thread.Sleep(TimeSpan.FromMinutes(minutes)); + Log($"Contract should remain stable for {Time.FormatDuration(mins)}."); + var endUtc = DateTime.UtcNow + mins; + while (DateTime.UtcNow < endUtc) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (proofWasMissed) + { + // We wait because we want to log calls to MarkProofAsMissing. + Thread.Sleep(periodDuration * 1.1); + Assert.Fail("Proof was missed."); + } + } - Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started)); + var minNumPeriod = (mins / periodDuration) - 1.0; + Log($"{numPeriods} periods elapsed. Expected at least {minNumPeriod} periods."); + Assert.That(numPeriods, Is.GreaterThanOrEqualTo(minNumPeriod)); + + var status = client.GetPurchaseStatus(purchase.PurchaseId); + if (status == null) throw new Exception("Purchase status not found"); + Assert.That(status.IsStarted || status.IsFinished); } - public void OnPeriodReport(PeriodReport report) + protected override void OnPeriod(PeriodReport report) { + numPeriods++; + // For each required proof, there should be a submit call. + var calls = GetSubmitProofCalls(report); foreach (var required in report.Required) { - var matchingCall = GetMatchingSubmitProofCall(report, required); - - Assert.That(matchingCall.FromAddress.ToLowerInvariant(), Is.EqualTo(required.Host.Address.ToLowerInvariant())); - Assert.That(matchingCall.Id.ToHex(), Is.EqualTo(required.SlotId.ToHex())); + var matchingCall = GetMatchingSubmitProofCall(calls, required); + if (matchingCall == null) + { + Log($"A proof was missed for {required.Describe()}. Failing test after a delay so chain events have time to log..."); + proofWasMissed = true; + } } // There can't be any calls to mark a proof as missed. @@ -72,23 +99,43 @@ namespace CodexReleaseTests.MarketTests } } - private SubmitProofFunction GetMatchingSubmitProofCall(PeriodReport report, PeriodRequiredProof required) + private SubmitProofFunction? GetMatchingSubmitProofCall(SubmitProofFunction[] calls, PeriodRequiredProof required) { - var submitCall = nameof(SubmitProofFunction); - var call = report.FunctionCalls.SingleOrDefault(f => f.Name == submitCall); - if (call == null) throw new Exception("Call to submitProof not found for " + required.Describe()); - var callObj = JsonConvert.DeserializeObject(call.Payload); - if (callObj == null) throw new Exception("Unable to deserialize call object"); - return callObj; + foreach (var call in calls) + { + if ( + call.Id.SequenceEqual(required.SlotId) && + call.FromAddress.ToLowerInvariant() == required.Host.Address.ToLowerInvariant() + ) + { + return call; + } + } + + return null; } - private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, int minutes) + private SubmitProofFunction[] GetSubmitProofCalls(PeriodReport report) + { + var submitCall = nameof(SubmitProofFunction); + var calls = report.FunctionCalls.Where(f => f.Name == submitCall).ToArray(); + var callObjs = calls.Select(call => JsonConvert.DeserializeObject(call.Payload)).ToArray(); + Log($"SubmitProof calls: {callObjs.Length}"); + foreach (var c in callObjs) + { + Log($" - slotId:{c.Id.ToHex()} host:{c.FromAddress}"); + } + + return callObjs!; + } + + private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes) { var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize)); var config = GetContracts().Deployment.Config; return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { - Duration = TimeSpan.FromMinutes(minutes) * 1.1, + Duration = minutes * 1.1, Expiry = TimeSpan.FromMinutes(8.0), MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes, NodeFailureTolerance = (uint)purchaseParams.Tolerance, diff --git a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs index f8cae455..07f621fb 100644 --- a/Tests/CodexReleaseTests/Utils/ChainMonitor.cs +++ b/Tests/CodexReleaseTests/Utils/ChainMonitor.cs @@ -11,21 +11,23 @@ namespace CodexReleaseTests.Utils private readonly ILog log; private readonly IGethNode gethNode; private readonly ICodexContracts contracts; + private readonly IPeriodMonitorEventHandler periodMonitorEventHandler; private readonly DateTime startUtc; private readonly TimeSpan updateInterval; private CancellationTokenSource cts = new CancellationTokenSource(); private Task worker = Task.CompletedTask; - public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) - : this(log, gethNode, contracts, startUtc, TimeSpan.FromSeconds(3.0)) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc) + : this(log, gethNode, contracts, periodMonitorEventHandler, startUtc, TimeSpan.FromSeconds(3.0)) { } - public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval) + public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc, TimeSpan updateInterval) { this.log = log; this.gethNode = gethNode; this.contracts = contracts; + this.periodMonitorEventHandler = periodMonitorEventHandler; this.startUtc = startUtc; this.updateInterval = updateInterval; } @@ -43,11 +45,9 @@ namespace CodexReleaseTests.Utils if (worker.Exception != null) throw worker.Exception; } - public IPeriodMonitorEventHandler PeriodMonitorEventHandler { get; set; } = new DoNothingPeriodMonitorEventHandler(); - private void Worker(Action onFailure) { - var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, PeriodMonitorEventHandler); + var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, periodMonitorEventHandler); Thread.Sleep(updateInterval); log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}"); diff --git a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs index 026a405f..43fcddae 100644 --- a/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs @@ -1,5 +1,6 @@ using CodexClient; using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; @@ -11,7 +12,7 @@ using Utils; namespace CodexReleaseTests.Utils { - public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest + public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest, IPeriodMonitorEventHandler { private MarketplaceHandle handle = null!; protected const int StartingBalanceTST = 1000; @@ -60,6 +61,9 @@ namespace CodexReleaseTests.Utils protected abstract TimeSpan HostAvailabilityMaxDuration { get; } protected virtual bool MonitorChainState { get; } = true; protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0); + protected virtual void OnPeriod(PeriodReport report) + { + } public ICodexNodeGroup StartHosts() { @@ -175,56 +179,6 @@ namespace CodexReleaseTests.Utils }); } - private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) - { - if (!MonitorChainState) return null; - - var result = new ChainMonitor(log, gethNode, contracts, startUtc); - result.Start(() => - { - Assert.Fail("Failure in chain monitor."); - }); - return result; - } - - private Retry GetBalanceAssertRetry() - { - return new Retry("AssertBalance", - maxTimeout: TimeSpan.FromMinutes(10.0), - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private Retry GetAvailabilitySpaceAssertRetry() - { - return new Retry("AssertAvailabilitySpace", - maxTimeout: HostBlockTTL * 3, - sleepAfterFail: TimeSpan.FromSeconds(10.0), - onFail: f => { }, - failFast: false); - } - - private TestToken GetTstBalance(ICodexNode node) - { - return GetContracts().GetTestTokenBalance(node); - } - - private TestToken GetTstBalance(EthAddress address) - { - return GetContracts().GetTestTokenBalance(address); - } - - private Ether GetEthBalance(ICodexNode node) - { - return GetGeth().GetEthBalance(node); - } - - private Ether GetEthBalance(EthAddress address) - { - return GetGeth().GetEthBalance(address); - } - public ICodexNodeGroup StartClients() { return StartClients(s => { }); @@ -253,6 +207,11 @@ namespace CodexReleaseTests.Utils ); } + public void OnPeriodReport(PeriodReport report) + { + OnPeriod(report); + } + public SlotFill[] GetOnChainSlotFills(IEnumerable possibleHosts, string purchaseId) { var fills = GetOnChainSlotFills(possibleHosts); @@ -351,6 +310,56 @@ namespace CodexReleaseTests.Utils } } + private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc) + { + if (!MonitorChainState) return null; + + var result = new ChainMonitor(log, gethNode, contracts, this, startUtc); + result.Start(() => + { + Assert.Fail("Failure in chain monitor."); + }); + return result; + } + + private Retry GetBalanceAssertRetry() + { + return new Retry("AssertBalance", + maxTimeout: TimeSpan.FromMinutes(10.0), + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private Retry GetAvailabilitySpaceAssertRetry() + { + return new Retry("AssertAvailabilitySpace", + maxTimeout: HostBlockTTL * 3, + sleepAfterFail: TimeSpan.FromSeconds(10.0), + onFail: f => { }, + failFast: false); + } + + private TestToken GetTstBalance(ICodexNode node) + { + return GetContracts().GetTestTokenBalance(node); + } + + private TestToken GetTstBalance(EthAddress address) + { + return GetContracts().GetTestTokenBalance(address); + } + + private Ether GetEthBalance(ICodexNode node) + { + return GetGeth().GetEthBalance(node); + } + + private Ether GetEthBalance(EthAddress address) + { + return GetGeth().GetEthBalance(address); + } + private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts) { var fills = GetOnChainSlotFills(hosts); @@ -465,6 +474,7 @@ namespace CodexReleaseTests.Utils var chanceOfDowntime = downtime / window; return 1.0f + (5.0f * chanceOfDowntime); } + public class SlotFill { public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)