Adds stability test

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -43,14 +43,14 @@ namespace CodexContractsPlugin.ChainMonitor
         private readonly IChainStateChangeHandler handler;
         private readonly bool doProofPeriodMonitoring;
 
-        public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring)
+        public ChainState(ILog log, IGethNode geth, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring, IPeriodMonitorEventHandler periodEventHandler)
         {
             this.log = new LogPrefixer(log, "(ChainState) ");
             this.contracts = contracts;
             handler = changeHandler;
             this.doProofPeriodMonitoring = doProofPeriodMonitoring;
             TotalSpan = new TimeRange(startUtc, startUtc);
-            PeriodMonitor = new PeriodMonitor(log, contracts, geth);
+            PeriodMonitor = new PeriodMonitor(log, contracts, geth, periodEventHandler);
         }
 
         public TimeRange TotalSpan { get; private set; }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs

@@ -10,19 +10,26 @@ using System.Reflection;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
+    public interface IPeriodMonitorEventHandler
+    {
+        void OnPeriodReport(PeriodReport report);
+    }
+
     public class PeriodMonitor
     {
         private readonly ILog log;
         private readonly ICodexContracts contracts;
         private readonly IGethNode geth;
+        private readonly IPeriodMonitorEventHandler eventHandler;
         private readonly List<PeriodReport> reports = new List<PeriodReport>();
         private CurrentPeriod? currentPeriod = null;
 
-        public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth)
+        public PeriodMonitor(ILog log, ICodexContracts contracts, IGethNode geth, IPeriodMonitorEventHandler eventHandler)
         {
             this.log = log;
             this.contracts = contracts;
             this.geth = geth;
+            this.eventHandler = eventHandler;
         }
 
         public void Update(DateTime eventUtc, IChainStateRequest[] requests)
@@ -55,9 +62,10 @@ namespace CodexContractsPlugin.ChainMonitor
                 {
                     var idx = Convert.ToInt32(slotIndex);
                     var host = request.Hosts.GetHost(idx);
+                    var slotId = contracts.GetSlotId(request.RequestId, slotIndex);
                     if (host != null)
                     {
-                        result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx));
+                        result.RequiredProofs.Add(new PeriodRequiredProof(host, request, idx, slotId));
                     }
                 }
             });
@@ -86,6 +94,8 @@ namespace CodexContractsPlugin.ChainMonitor
 
             report.Log(log);
             reports.Add(report);
+
+            eventHandler.OnPeriodReport(report);
         }
 
         private void ForEachActiveSlot(IChainStateRequest[] requests, Action<IChainStateRequest, ulong> action)
@@ -100,6 +110,13 @@ namespace CodexContractsPlugin.ChainMonitor
         }
     }
 
+    public class DoNothingPeriodMonitorEventHandler : IPeriodMonitorEventHandler
+    {
+        public void OnPeriodReport(PeriodReport report)
+        {
+        }
+    }
+
     public class CallReporter
     {
         private readonly List<FunctionCallReport> reports;

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodRequiredProof.cs

@@ -5,16 +5,18 @@ namespace CodexContractsPlugin.ChainMonitor
 {
     public class PeriodRequiredProof
     {
-        public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex)
+        public PeriodRequiredProof(EthAddress host, IChainStateRequest request, int slotIndex, byte[] slotId)
         {
             Host = host;
             Request = request;
             SlotIndex = slotIndex;
+            SlotId = slotId;
         }
 
         public EthAddress Host { get; }
         public IChainStateRequest Request { get; }
         public int SlotIndex { get; }
+        public byte[] SlotId { get; }
 
         public string Describe()
         {

Tests/CodexReleaseTests/MarketTests/StabilityTest.cs

@@ -0,0 +1,100 @@
+using CodexClient;
+using CodexContractsPlugin.ChainMonitor;
+using CodexContractsPlugin.Marketplace;
+using CodexReleaseTests.Utils;
+using Nethereum.Hex.HexConvertors.Extensions;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class StabilityTest : MarketplaceAutoBootstrapDistTest, IPeriodMonitorEventHandler
+    {
+        #region Setup
+
+        private readonly PurchaseParams purchaseParams = new PurchaseParams(
+            nodes: 4,
+            tolerance: 2,
+            uploadFilesize: 32.MB()
+        );
+
+        public StabilityTest()
+        {
+            Assert.That(purchaseParams.Nodes, Is.LessThan(NumberOfHosts));
+        }
+
+        protected override int NumberOfHosts => 6;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => purchaseParams.SlotSize.Multiply(1.1); // Each host can hold 1 slot.
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(5.0);
+
+        #endregion
+
+        [Test]
+        [Combinatorial]
+        public void Stability(
+            [Values(10, 120)] int minutes)
+        {
+            Assert.That(HostAvailabilityMaxDuration, Is.GreaterThan(TimeSpan.FromMinutes(minutes * 1.1)));
+
+            GetChainMonitor().PeriodMonitorEventHandler = this;
+
+            StartHosts();
+            StartValidator();
+            var client = StartClients().Single();
+            var purchase = CreateStorageRequest(client, minutes);
+
+            Log($"Contract should remain stable for {minutes} minutes.");
+            Thread.Sleep(TimeSpan.FromSeconds(minutes));
+
+            Assert.That(client.GetPurchaseStatus(purchase.PurchaseId)?.State, Is.EqualTo(StoragePurchaseState.Started));
+        }
+
+        public void OnPeriodReport(PeriodReport report)
+        {
+            // For each required proof, there should be a submit call.
+            foreach (var required in report.Required)
+            {
+                var matchingCall = GetMatchingSubmitProofCall(report, required);
+
+                Assert.That(matchingCall.FromAddress.ToLowerInvariant(), Is.EqualTo(required.Host.Address.ToLowerInvariant()));
+                Assert.That(matchingCall.Id.ToHex(), Is.EqualTo(required.SlotId.ToHex()));
+            }
+
+            // There can't be any calls to mark a proof as missed.
+            foreach (var call in report.FunctionCalls)
+            {
+                var missedCall = nameof(MarkProofAsMissingFunction);
+                Assert.That(call.Name, Is.Not.EqualTo(missedCall));
+            }
+        }
+
+        private SubmitProofFunction GetMatchingSubmitProofCall(PeriodReport report, PeriodRequiredProof required)
+        {
+            var submitCall = nameof(SubmitProofFunction);
+            var call = report.FunctionCalls.SingleOrDefault(f => f.Name == submitCall);
+            if (call == null) throw new Exception("Call to submitProof not found for " + required.Describe());
+            var callObj = JsonConvert.DeserializeObject<SubmitProofFunction>(call.Payload);
+            if (callObj == null) throw new Exception("Unable to deserialize call object");
+            return callObj;
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client, int minutes)
+        {
+            var cid = client.UploadFile(GenerateTestFile(purchaseParams.UploadFilesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = TimeSpan.FromMinutes(minutes) * 1.1,
+                Expiry = TimeSpan.FromMinutes(10.0),
+                MinRequiredNumberOfNodes = (uint)purchaseParams.Nodes,
+                NodeFailureTolerance = (uint)purchaseParams.Tolerance,
+                PricePerBytePerSecond = 10.TstWei(),
+                ProofProbability = 1, // One proof every period. Free slot as quickly as possible.
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+    }
+}

Tests/CodexReleaseTests/Utils/ChainMonitor.cs

@@ -43,9 +43,11 @@ namespace CodexReleaseTests.Utils
             if (worker.Exception != null) throw worker.Exception;
         }
 
+        public IPeriodMonitorEventHandler PeriodMonitorEventHandler { get; set; } = new DoNothingPeriodMonitorEventHandler();
+
         private void Worker(Action onFailure)
         {
-            var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, doProofPeriodMonitoring: true);
+            var state = new ChainState(log, gethNode, contracts, new DoNothingThrowingChainEventHandler(), startUtc, true, PeriodMonitorEventHandler);
             Thread.Sleep(updateInterval);
 
             log.Log($"Chain monitoring started. Update interval: {Time.FormatDuration(updateInterval)}");

Tests/CodexReleaseTests/Utils/MarketplaceAutoBootstrapDistTest.cs

@@ -42,6 +42,12 @@ namespace CodexReleaseTests.Utils
             return handle.Contracts;
         }
 
+        protected ChainMonitor GetChainMonitor()
+        {
+            if (handle.ChainMonitor == null) throw new Exception($"Make sure {nameof(MonitorChainState)} is set to true.");
+            return handle.ChainMonitor;
+        }
+
         protected TimeSpan GetPeriodDuration()
         {
             var config = GetContracts().Deployment.Config;

Tools/MarketInsights/AverageHistory.cs

@@ -19,7 +19,7 @@ namespace MarketInsights
             this.appState = appState;
             this.maxContributions = maxContributions;
             chainState = new ChainState(appState.Log, geth, contracts, mux, appState.Config.HistoryStartUtc,
-                doProofPeriodMonitoring: false);
+                doProofPeriodMonitoring: false, new DoNothingPeriodMonitorEventHandler());
         }
 
         public MarketTimeSegment[] Segments { get; private set; } = Array.Empty<MarketTimeSegment>();

Tools/TestNetRewarder/Processor.cs

@@ -29,7 +29,7 @@ namespace TestNetRewarder
             eventsFormatter = new EventsFormatter(config, contracts.Deployment.Config);
 
             chainState = new ChainState(log, geth, contracts, eventsFormatter, config.HistoryStartUtc,
-                doProofPeriodMonitoring: config.ShowProofPeriodReports > 0);
+                doProofPeriodMonitoring: config.ShowProofPeriodReports > 0, new DoNothingPeriodMonitorEventHandler());
         }
 
         public async Task Initialize()

Tools/TraceContract/ChainTracer.cs

@@ -63,7 +63,7 @@ namespace TraceContract
             var utc = request.Block.Utc.AddMinutes(-1.0);
             var tracker = new ChainRequestTracker(output, input.PurchaseId);
             var ignoreLog = new NullLog();
-            var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false);
+            var chainState = new ChainState(ignoreLog, geth, contracts, tracker, utc, false, new DoNothingPeriodMonitorEventHandler());
 
             var atNow = false;
             while (!tracker.IsFinished && !atNow)