Sets up asserting of balances in case of failed contract.

Framework/KubernetesWorkflow/LogHandler.cs

@@ -40,6 +40,10 @@ namespace KubernetesWorkflow
 
         protected override void ProcessLine(string line)
         {
+            // This line is not useful and has no topic so we can't filter it with
+            // normal log-level controls.
+            if (line.Contains("Received JSON-RPC response") && !line.Contains("topics=")) return;
+
             LogFile.WriteRaw(line);
         }
     }

Framework/Utils/EthAddress.cs

@@ -30,5 +30,15 @@
         {
             return Address;
         }
+
+        public static bool operator ==(EthAddress a, EthAddress b)
+        {
+            return a.Address == b.Address;
+        }
+
+        public static bool operator !=(EthAddress a, EthAddress b)
+        {
+            return a.Address != b.Address;
+        }
     }
 }

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -11,7 +11,7 @@ namespace CodexClient
         private readonly ILog log;
         private readonly IHttpFactory httpFactory;
         private readonly IProcessControl processControl;
-        private ICodexInstance instance;
+        private readonly ICodexInstance instance;
         private readonly Mapper mapper = new Mapper();
 
         public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
@@ -25,8 +25,6 @@ namespace CodexClient
         public void Stop(bool waitTillStopped)
         {
             processControl.Stop(waitTillStopped);
-            // Prevents accidental use after stop:
-            instance = null!;
         }
 
         public IDownloadedLog DownloadLog(string additionalName = "")

ProjectPlugins/CodexClient/CodexClient.csproj

@@ -30,6 +30,7 @@
     <ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
     <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
     <ProjectReference Include="..\..\Framework\WebUtils\WebUtils.csproj" />
+    <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
   </ItemGroup>
 
 </Project>

ProjectPlugins/CodexClient/StoragePurchaseContract.cs

@@ -1,4 +1,5 @@
 using CodexClient.Hooks;
+using CodexContractsPlugin.Marketplace;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -14,7 +15,7 @@ namespace CodexClient
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
         void WaitForStorageContractFinished();
-        void WaitForContractFailed();
+        void WaitForContractFailed(MarketplaceConfig config);
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -99,7 +100,7 @@ namespace CodexClient
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
         }
 
-        public void WaitForContractFailed()
+        public void WaitForContractFailed(MarketplaceConfig config)
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -107,9 +108,27 @@ namespace CodexClient
             }
             var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
             var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            var minTimeout = TimeNeededToFailEnoughProofsToFreeASlot(config);
+
+            if (timeout < minTimeout)
+            {
+                throw new ArgumentOutOfRangeException(
+                    $"Test is misconfigured. Assuming a proof is required every period, it will take {Time.FormatDuration(minTimeout)} " +
+                    $"to fail enough proofs for a slot to be freed. But, the storage contract will complete in {Time.FormatDuration(timeout)}. " +
+                    $"Increase the duration."
+                );
+            }
+
             WaitForStorageContractState(timeout, "failed");
         }
 
+        private TimeSpan TimeNeededToFailEnoughProofsToFreeASlot(MarketplaceConfig config)
+        {
+            var numMissedProofsRequiredForFree = config.Collateral.MaxNumberOfSlashes;
+            var timePerProof = TimeSpan.FromSeconds(config.Proofs.Period);
+            return timePerProof * (numMissedProofsRequiredForFree + 1);
+        }
+
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
         {
             var waitStart = DateTime.UtcNow;

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -1,6 +1,9 @@
 using CodexClient;
+using CodexContractsPlugin.ChainMonitor;
 using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
 using NUnit.Framework;
+using System.Numerics;
 using Utils;
 
 namespace CodexReleaseTests.MarketTests
@@ -21,7 +24,7 @@ namespace CodexReleaseTests.MarketTests
         {
             var hosts = StartHosts();
             var client = StartClients().Single();
-            StartValidator();
+            var validator = StartValidator();
 
             var request = CreateStorageRequest(client);
 
@@ -32,32 +35,97 @@ namespace CodexReleaseTests.MarketTests
             AssertContractSlotsAreFilledByHosts(request, hosts);
 
             hosts.Stop(waitTillStopped: true);
+            
+            var config = GetContracts().Deployment.Config;
+            request.WaitForContractFailed(config);
 
-            WaitForSlotFreedEvents();
-            AssertProofMissedReports();
+            var frees = GetOnChainSlotFrees(hosts);
+            Assert.That(frees.Length, Is.EqualTo(
+                request.Purchase.MinRequiredNumberOfNodes - request.Purchase.NodeFailureTolerance));
 
-            request.WaitForContractFailed();
+            var periodReports = GetPeriodMonitorReports();
+            var missedProofs = periodReports.Reports.SelectMany(r => r.MissedProofs).ToArray();
+            AssertEnoughProofsWereMissedForSlotFree(frees, missedProofs, config);
+
+            AssertClientPaidNothing(client);
+            AssertValidatorWasPaidPerMissedProof(validator, request, missedProofs, config);
+            AssertHostCollateralWasBurned(hosts, request);
         }
 
-        private void WaitForSlotFreedEvents()
+        private void AssertClientPaidNothing(ICodexNode client)
         {
-            Log(nameof(WaitForSlotFreedEvents));
+            AssertTstBalance(client, StartingBalanceTST.Tst(), "Client should not have paid for failed contract.");
+        }
 
-            var start = DateTime.UtcNow;
-            var timeout = CalculateContractFailTimespan();
+        private void AssertValidatorWasPaidPerMissedProof(ICodexNode validator, IStoragePurchaseContract request, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+        {
+            var rewardPerMissedProof = GetValidatorRewardPerMissedProof(request, config);
+            var totalValidatorReward = rewardPerMissedProof * missedProofs.Length;
 
-            while (DateTime.UtcNow < start + timeout)
+            AssertTstBalance(validator, StartingBalanceTST.Tst() + totalValidatorReward, $"Validator is rewarded per slot marked as missing. " +
+                $"numberOfMissedProofs: {missedProofs.Length} rewardPerMissedProof: {rewardPerMissedProof}");
+        }
+
+        private TestToken GetCollatoralPerSlot(IStoragePurchaseContract request)
+        {
+            var slotSize = new ByteSize(Convert.ToInt64(request.GetStatus()!.Request.Ask.SlotSize));
+            return new TestToken(request.Purchase.CollateralPerByte.TstWei * slotSize.SizeInBytes);
+        }
+
+        private void AssertHostCollateralWasBurned(ICodexNodeGroup hosts, IStoragePurchaseContract request)
+        {
+            var slotFills = GetOnChainSlotFills(hosts);
+            foreach (var host in hosts)
             {
-                var events = GetContracts().GetEvents(GetTestRunTimeRange());
-                var slotFreed = events.GetSlotFreedEvents();
-                if (slotFreed.Length == NumberOfSlots)
-                {
-                    Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
-                    return;
-                }
-                GetContracts().WaitUntilNextPeriod();
+                AssertHostCollateralWasBurned(host, slotFills, request);
             }
-            Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private void AssertHostCollateralWasBurned(ICodexNode host, SlotFill[] slotFills, IStoragePurchaseContract request)
+        {
+            // In case of a failed contract, the entire slotColateral is lost.
+            var filledByHost = slotFills.Where(f => f.Host.EthAddress == host.EthAddress).ToArray();
+            var numSlotsOfHost = filledByHost.Length;
+            var collatoralPerSlot = GetCollatoralPerSlot(request);
+            var totalCost = collatoralPerSlot * numSlotsOfHost;
+
+            AssertTstBalance(host, StartingBalanceTST.Tst() - totalCost, $"Host has lost collateral for each slot. " +
+                $"numberOfSlotsByHost: {numSlotsOfHost} collateralPerSlot: {collatoralPerSlot}");
+        }
+
+        private TestToken GetValidatorRewardPerMissedProof(IStoragePurchaseContract request, MarketplaceConfig config)
+        {
+            var collatoralPerSlot = GetCollatoralPerSlot(request);
+            var slashPercentage = config.Collateral.SlashPercentage;
+            var validatorRewardPercentage = config.Collateral.ValidatorRewardPercentage;
+
+            var rewardPerMissedProof =
+                PercentageOf(
+                    PercentageOf(collatoralPerSlot, slashPercentage),
+                    validatorRewardPercentage);
+
+            return rewardPerMissedProof;
+        }
+
+        private TestToken PercentageOf(TestToken value, byte percentage)
+        {
+            var p = new BigInteger(percentage);
+            return new TestToken((value.TstWei * p) / 100);
+        }
+
+        private void AssertEnoughProofsWereMissedForSlotFree(SlotFree[] frees, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+        {
+            foreach (var free in frees)
+            {
+                AssertEnoughProofsWereMissedForSlotFree(free, missedProofs, config);
+            }
+        }
+
+        private void AssertEnoughProofsWereMissedForSlotFree(SlotFree free, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+        {
+            var missedByHost = missedProofs.Where(p => p.Host != null && p.Host.Address == free.Host.EthAddress.Address).ToArray();
+            var maxNumMissedProofsBeforeFreeSlot = config.Collateral.MaxNumberOfSlashes;
+            Assert.That(missedByHost.Length, Is.EqualTo(maxNumMissedProofsBeforeFreeSlot));
         }
 
         private TimeSpan CalculateContractFailTimespan()
@@ -92,8 +160,8 @@ namespace CodexReleaseTests.MarketTests
             var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
             return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
             {
-                Duration = TimeSpan.FromHours(1.0),
-                Expiry = TimeSpan.FromHours(0.2),
+                Duration = TimeSpan.FromMinutes(10.0),
+                Expiry = TimeSpan.FromMinutes(5.0),
                 MinRequiredNumberOfNodes = NumberOfSlots,
                 NodeFailureTolerance = 1,
                 PricePerBytePerSecond = pricePerBytePerSecond,

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -68,8 +68,8 @@ namespace CodexReleaseTests.MarketTests
             var hosts = StartCodex(NumberOfHosts, s => s
                 .WithName("host")
                 .WithBlockTTL(HostBlockTTL)
-                .WithBlockMaintenanceNumber(100000)
-                .WithBlockMaintenanceInterval(HostBlockTTL / 4)
+                .WithBlockMaintenanceNumber(100)
+                .WithBlockMaintenanceInterval(HostBlockTTL / 2)
                 .EnableMarketplace(GetGeth(), GetContracts(), m => m
                     .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
                     .AsStorageNode()
@@ -236,6 +236,29 @@ namespace CodexReleaseTests.MarketTests
             }).ToArray();
         }
 
+        public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts, string purchaseId)
+        {
+            var fills = GetOnChainSlotFrees(possibleHosts);
+            return fills.Where(f => f
+                .SlotFreedEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
+                .ToArray();
+        }
+
+        public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var fills = GetOnChainSlotFills(possibleHosts);
+            var frees = events.GetSlotFreedEvents();
+            return frees.Select(f =>
+            {
+                var matchingFill = fills.Single(fill => fill.SlotFilledEvent.RequestId == f.RequestId &&
+                    fill.SlotFilledEvent.SlotIndex == f.SlotIndex);
+
+                return new SlotFree(f, matchingFill.Host);
+
+            }).ToArray();
+        }
+
         protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
         {
             var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts);
@@ -359,13 +382,6 @@ namespace CodexReleaseTests.MarketTests
             }, description);
         }
 
-        protected void AssertEnoughProofMissedForSlotFree(ICodexNodeGroup hosts)
-        {
-            var slotFills = GetOnChainSlotFills(hosts);
-
-            todo for each filled slot, there should be enough proofs missed to trigger the slot-free event.
-        }
-
         public class SlotFill
         {
             public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
@@ -378,6 +394,18 @@ namespace CodexReleaseTests.MarketTests
             public ICodexNode Host { get; }
         }
 
+        public class SlotFree
+        {
+            public SlotFree(SlotFreedEventDTO slotFreedEvent, ICodexNode host)
+            {
+                SlotFreedEvent = slotFreedEvent;
+                Host = host;
+            }
+
+            public SlotFreedEventDTO SlotFreedEvent { get; }
+            public ICodexNode Host { get; }
+        }
+
         private class MarketplaceHandle
         {
             public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor monitor)

Tests/CodexReleaseTests/Parallelism.cs

@@ -1,6 +1,6 @@
 using NUnit.Framework;
 
-[assembly: LevelOfParallelism(10)]
+[assembly: LevelOfParallelism(1)]
 namespace CodexReleaseTests.DataTests
 {
 }

Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs

@@ -0,0 +1,46 @@
+using GethPlugin;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EthAddressEqualityTests
+    {
+        [Test]
+        [Combinatorial]
+        public void Equal(
+            [Values(1, 2, 3, 4, 5)] int runs
+        )
+        {
+            var account = EthAccountGenerator.GenerateNew();
+
+            var str = account.EthAddress.Address;
+
+            var addr = new EthAddress(str);
+
+            Assert.That(addr, Is.EqualTo(account.EthAddress));
+            Assert.That(addr == account.EthAddress);
+            Assert.That(!(addr != account.EthAddress));
+        }
+
+        [Test]
+        [Combinatorial]
+        public void NotEqual(
+            [Values(1, 2, 3, 4, 5)] int runs
+        )
+        {
+            var account1 = EthAccountGenerator.GenerateNew();
+            var account2 = EthAccountGenerator.GenerateNew();
+
+            Assert.That(account1.EthAddress, Is.Not.EqualTo(account2.EthAddress));
+            Assert.That(account1.EthAddress != account2.EthAddress);
+            Assert.That(!(account1.EthAddress == account2.EthAddress));
+        }
+    }
+}