Cleanup stability test. Add test to check when proof is required

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodReport.cs

@@ -19,9 +19,10 @@ namespace CodexContractsPlugin.ChainMonitor
         public void Log(ILog log)
         {
             log.Log($"Period report: {Period}");
+            log.Log($" - Proofs required: {Required.Length}");
             foreach (var r in Required)
             {
-                log.Log($"  Required: {r.Describe()}");
+                log.Log($"   - {r.Describe()}");
             }
             log.Log($" - Calls: {FunctionCalls.Length}");
             foreach (var f in FunctionCalls)

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs

@@ -20,7 +20,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         public string Describe()
         {
-            return $"{Request.RequestId.ToHex()} slotIndex:{SlotIndex} by {Host}";
+            return $"{Request.RequestId.ToHex()} slotId:{SlotId.ToHex()} slotIndex:{SlotIndex} by {Host}";
         }
     }
 }

Tests/CodexReleaseTests/MarketTests/IsProofRequiredTest.cs

@@ -0,0 +1,95 @@
+using CodexClient;
+using CodexContractsPlugin.ChainMonitor;
+using CodexReleaseTests.Utils;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class IsProofRequiredTest : MarketplaceAutoBootstrapDistTest
+    {
+        #region Setup
+
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 4,
+            tolerance: 2,
+            uploadFilesize: 32.MB()
+        );
+
+        public IsProofRequiredTest()
+        {
+            Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
+        }
+
+        protected override int NumberOfHosts => 6;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
+
+        #endregion
+
+        [Test]
+        public void IsProofRequired()
+        {
+            var mins = TimeSpan.FromMinutes(10.0);
+
+            StartHosts();
+            StartValidator();
+            var client = StartClients().Single();
+            var purchase = CreateStorageRequest(client, mins);
+            purchase.WaitForStorageContractStarted();
+
+            var requestId = purchase.PurchaseId.HexToByteArray();
+            var numSlots = purchaseParams.Nodes;
+            //var map = new Dictionary<ulong, List<int>>();
+
+            Log($"Checking IsProofRequired every second for {Time.FormatDuration(mins)}.");
+            var endUtc = DateTime.UtcNow + mins;
+            while (DateTime.UtcNow < endUtc)
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(1));
+                var requiredSlotIndices = new List<int>();
+                for (var i = 0; i < numSlots; i++)
+                {
+                    if (GetContracts().IsProofRequired(requestId, i)) requiredSlotIndices.Add(i);
+                }
+
+                var periodNumber = GetContracts().GetPeriodNumber(DateTime.UtcNow);
+                var blockNumber = GetGeth().GetSyncedBlockNumber();
+                Log($"[{blockNumber?.ToString().PadLeft(4, '0')}]" +
+                    $"{periodNumber.ToString().PadLeft(12, '0')} => " +
+                    $"{string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}");
+
+                //var num = currentPeriod.PeriodNumber;
+                //if (!map.ContainsKey(num))
+                //{
+                //    map.Add(num, requiredSlotIndices);
+                //    Log($"Period {num} = required proof for slot indices {string.Join(",", requiredSlotIndices.Select(i => i.ToString()))}");
+                //}
+                //else
+                //{
+                //    var a = map[num];
+                //    CollectionAssert.AreEquivalent(a, requiredSlotIndices);
+                //}
+            }
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes)
+        {
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = minutes * 1.1,
+                Expiry = TimeSpan.FromMinutes(8.0),
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
+                PricePerBytePerSecond = 10.TstWei(),
+                ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/StabilityTest.cs

@@ -10,7 +10,7 @@ using Utils;
 namespace CodexReleaseTests.MarketTests
 {
     [TestFixture]
-    public class StabilityTest : MarketplaceAutoBootstrapDistTest, IPeriodMonitorEventHandler
+    public class StabilityTest : MarketplaceAutoBootstrapDistTest
     {
         #region Setup
 
@@ -32,36 +32,63 @@ namespace CodexReleaseTests.MarketTests
 
         #endregion
 
+        private int numPeriods = 0;
+        private bool proofWasMissed = false;
+
         [Test]
         [Combinatorial]
         public void Stability(
             [Values(10, 120)] int minutes)
         {
-            Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(TimeSpan.FromMinutes(minutes * 1.1)));
+            var mins = TimeSpan.FromMinutes(minutes);
+            var periodDuration = GetContracts().Deployment.Config.PeriodDuration;
+            Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(mins * 1.1));
 
-            GetChainMonitor().PeriodMonitorEventHandler = this;
+            numPeriods = 0;
+            proofWasMissed = false;
 
             StartHosts();
             StartValidator();
             var client = StartClients().Single();
-            var purchase = CreateStorageRequest(client, minutes);
+            var purchase = CreateStorageRequest(client, mins);
             purchase.WaitForStorageContractStarted();
 
-            Log($"Contract should remain stable for {minutes} minutes.");
-            Thread.Sleep(TimeSpan.FromMinutes(minutes));
+            Log($"Contract should remain stable for {Time.FormatDuration(mins)}.");
+            var endUtc = DateTime.UtcNow + mins;
+            while (DateTime.UtcNow < endUtc)
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(10));
+                if (proofWasMissed)
+                {
+                    // We wait because we want to log calls to MarkProofAsMissing.
+                    Thread.Sleep(periodDuration * 1.1);
+                    Assert.Fail("Proof was missed.");
+                }
+            }
 
-            Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started));
+            var minNumPeriod = (mins / periodDuration) - 1.0;
+            Log($"{numPeriods} periods elapsed. Expected at least {minNumPeriod} periods.");
+            Assert.That(numPeriods, Is.GreaterThanOrEqualTo(minNumPeriod));
+
+            var status = client.GetPurchaseStatus(purchase.PurchaseId);
+            if (status == null) throw new Exception("Purchase status not found");
+            Assert.That(status.IsStarted || status.IsFinished);
         }
 
-        public void OnPeriodReport(PeriodReport report)
+        protected override void OnPeriod(PeriodReport report)
         {
+            numPeriods++;
+
             // For each required proof, there should be a submit call.
+            var calls = GetSubmitProofCalls(report);
             foreach (var required in report.Required)
             {
-                var matchingCall = GetMatchingSubmitProofCall(report, required);
-
-                Assert.That(matchingCall.FromAddress.ToLowerInvariant(), Is.EqualTo(required.Host.Address.ToLowerInvariant()));
-                Assert.That(matchingCall.Id.ToHex(), Is.EqualTo(required.SlotId.ToHex()));
+                var matchingCall = GetMatchingSubmitProofCall(calls, required);
+                if (matchingCall == null)
+                {
+                    Log($"A proof was missed for {required.Describe()}. Failing test after a delay so chain events have time to log...");
+                    proofWasMissed = true;
+                }
             }
 
             // There can't be any calls to mark a proof as missed.
@@ -72,23 +99,43 @@ namespace CodexReleaseTests.MarketTests
             }
         }
 
-        private SubmitProofFunction GetMatchingSubmitProofCall(PeriodReport report, PeriodRequiredProof required)
+        private SubmitProofFunction? GetMatchingSubmitProofCall(SubmitProofFunction[] calls, PeriodRequiredProof required)
         {
-            var submitCall = nameof(SubmitProofFunction);
-            var call = report.FunctionCalls.SingleOrDefault(f => f.Name == submitCall);
-            if (call == null) throw new Exception("Call to submitProof not found for " + required.Describe());
-            var callObj = JsonConvert.DeserializeObject<SubmitProofFunction>(call.Payload);
-            if (callObj == null) throw new Exception("Unable to deserialize call object");
-            return callObj;
+            foreach (var call in calls)
+            {
+                if (
+                    call.Id.SequenceEqual(required.SlotId) &&
+                    call.FromAddress.ToLowerInvariant() == required.Host.Address.ToLowerInvariant()
+                )
+                {
+                    return call;
+                }
+            }
+
+            return null;
         }
 
-        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, int minutes)
+        private SubmitProofFunction[] GetSubmitProofCalls(PeriodReport report)
+        {
+            var submitCall = nameof(SubmitProofFunction);
+            var calls = report.FunctionCalls.Where(f => f.Name == submitCall).ToArray();
+            var callObjs = calls.Select(call => JsonConvert.DeserializeObject<SubmitProofFunction>(call.Payload)).ToArray();
+            Log($"SubmitProof calls: {callObjs.Length}");
+            foreach (var c in callObjs)
+            {
+                Log($" - slotId:{c.Id.ToHex()} host:{c.FromAddress}");
+            }
+
+            return callObjs!;
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, TimeSpan minutes)
         {
             var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
             var config = GetContracts().Deployment.Config;
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
-                Duration = TimeSpan.FromMinutes(minutes) * 1.1,
+                Duration = minutes * 1.1,
                 Expiry = TimeSpan.FromMinutes(8.0),
                 MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
                 NodeFailureTolerance = (uint)purchaseParams.Tolerance,

Tests/CodexReleaseTests/Utils/ChainMonitor.cs

@@ -11,21 +11,23 @@ namespace CodexReleaseTests.Utils
         private readonly ILog log;
         private readonly IGethNode gethNode;
         private readonly ICodexContracts contracts;
+        private readonly IPeriodMonitorEventHandler periodMonitorEventHandler;
         private readonly DateTime startUtc;
         private readonly TimeSpan updateInterval;
         private CancellationTokenSource cts = new CancellationTokenSource();
         private Task worker = Task.CompletedTask;
 
-        public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc)
-            : this(log, gethNode, contracts, startUtc, TimeSpan.FromSeconds(3.0))
+        public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc)
+            : this(log, gethNode, contracts, periodMonitorEventHandler, startUtc, TimeSpan.FromSeconds(3.0))
         {
         }
 
-        public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc, TimeSpan updateInterval)
+        public ChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, IPeriodMonitorEventHandler periodMonitorEventHandler, DateTime startUtc, TimeSpan updateInterval)
         {
             this.log = log;
             this.gethNode = gethNode;
             this.contracts = contracts;
+            this.periodMonitorEventHandler = periodMonitorEventHandler;
             this.startUtc = startUtc;
             this.updateInterval = updateInterval;
         }
@@ -43,11 +45,9 @@ namespace CodexReleaseTests.Utils
             if (worker.Exception != null) throw worker.Exception;
         }
 
-        public IPeriodMonitorEventHandler PeriodMonitorEventHandler { get; set; } = new DoNothingPeriodMonitorEventHandler();
-
         private void Worker(Action onFailure)
         {
-            var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, PeriodMonitorEventHandler);
+            var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, periodMonitorEventHandler);
             Thread.Sleep(updateInterval);
 
             log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}");

Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs

@@ -1,5 +1,6 @@
 using CodexClient;
 using CodexContractsPlugin;
+using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
 using CodexPlugin;
 using CodexTests;
@@ -11,7 +12,7 @@ using Utils;
 
 namespace CodexReleaseTests.Utils
 {
-    public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
+    public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest, IPeriodMonitorEventHandler
     {
         private MarketplaceHandle handle = null!;
         protected const int StartingBalanceTST = 1000;
@@ -60,6 +61,9 @@ namespace CodexReleaseTests.Utils
         protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
         protected virtual bool MonitorChainState { get; } = true;
         protected TimeSpan HostBlockTTL { get; } = TimeSpan.FromMinutes(1.0);
+        protected virtual void OnPeriod(PeriodReport report)
+        {
+        }
 
         public ICodexNodeGroup StartHosts()
         {
@@ -175,56 +179,6 @@ namespace CodexReleaseTests.Utils
             });
         }
 
-        private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc)
-        {
-            if (!MonitorChainState) return null;
-
-            var result = new ChainMonitor(log, gethNode, contracts, startUtc);
-            result.Start(() =>
-            {
-                Assert.Fail("Failure in chain monitor.");
-            });
-            return result;
-        }
-
-        private Retry GetBalanceAssertRetry()
-        {
-            return new Retry("AssertBalance",
-                maxTimeout: TimeSpan.FromMinutes(10.0),
-                sleepAfterFail: TimeSpan.FromSeconds(10.0),
-                onFail: f => { },
-                failFast: false);
-        }
-
-        private Retry GetAvailabilitySpaceAssertRetry()
-        {
-            return new Retry("AssertAvailabilitySpace",
-                maxTimeout: HostBlockTTL * 3,
-                sleepAfterFail: TimeSpan.FromSeconds(10.0),
-                onFail: f => { },
-                failFast: false);
-        }
-
-        private TestToken GetTstBalance(ICodexNode node)
-        {
-            return GetContracts().GetTestTokenBalance(node);
-        }
-
-        private TestToken GetTstBalance(EthAddress address)
-        {
-            return GetContracts().GetTestTokenBalance(address);
-        }
-
-        private Ether GetEthBalance(ICodexNode node)
-        {
-            return GetGeth().GetEthBalance(node);
-        }
-
-        private Ether GetEthBalance(EthAddress address)
-        {
-            return GetGeth().GetEthBalance(address);
-        }
-
         public ICodexNodeGroup StartClients()
         {
             return StartClients(s => { });
@@ -253,6 +207,11 @@ namespace CodexReleaseTests.Utils
             );
         }
 
+        public void OnPeriodReport(PeriodReport report)
+        {
+            OnPeriod(report);
+        }
+
         public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts, string purchaseId)
         {
             var fills = GetOnChainSlotFills(possibleHosts);
@@ -351,6 +310,56 @@ namespace CodexReleaseTests.Utils
             }
         }
 
+        private ChainMonitor? SetupChainMonitor(ILog log, IGethNode gethNode, ICodexContracts contracts, DateTime startUtc)
+        {
+            if (!MonitorChainState) return null;
+
+            var result = new ChainMonitor(log, gethNode, contracts, this, startUtc);
+            result.Start(() =>
+            {
+                Assert.Fail("Failure in chain monitor.");
+            });
+            return result;
+        }
+
+        private Retry GetBalanceAssertRetry()
+        {
+            return new Retry("AssertBalance",
+                maxTimeout: TimeSpan.FromMinutes(10.0),
+                sleepAfterFail: TimeSpan.FromSeconds(10.0),
+                onFail: f => { },
+                failFast: false);
+        }
+
+        private Retry GetAvailabilitySpaceAssertRetry()
+        {
+            return new Retry("AssertAvailabilitySpace",
+                maxTimeout: HostBlockTTL * 3,
+                sleepAfterFail: TimeSpan.FromSeconds(10.0),
+                onFail: f => { },
+                failFast: false);
+        }
+
+        private TestToken GetTstBalance(ICodexNode node)
+        {
+            return GetContracts().GetTestTokenBalance(node);
+        }
+
+        private TestToken GetTstBalance(EthAddress address)
+        {
+            return GetContracts().GetTestTokenBalance(address);
+        }
+
+        private Ether GetEthBalance(ICodexNode node)
+        {
+            return GetGeth().GetEthBalance(node);
+        }
+
+        private Ether GetEthBalance(EthAddress address)
+        {
+            return GetGeth().GetEthBalance(address);
+        }
+
         private TestToken GetContractFinalCost(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
         {
             var fills = GetOnChainSlotFills(hosts);
@@ -465,6 +474,7 @@ namespace CodexReleaseTests.Utils
             var chanceOfDowntime = downtime / window;
             return 1.0f + (5.0f * chanceOfDowntime);
         }
+
         public class SlotFill
         {
             public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)