Sets up repair test

Framework/Utils/EthAccount.cs

@@ -1,7 +1,7 @@
 namespace Utils
 {
     [Serializable]
-    public class EthAccount
+    public class EthAccount : IComparable<EthAccount>
     {
         public EthAccount(EthAddress ethAddress, string privateKey)
         {
@@ -12,9 +12,34 @@
         public EthAddress EthAddress { get; }
         public string PrivateKey { get; }
 
+        public int CompareTo(EthAccount? other)
+        {
+            return PrivateKey.CompareTo(other!.PrivateKey);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is EthAccount token && PrivateKey == token.PrivateKey;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(PrivateKey);
+        }
+
         public override string ToString()
         {
             return EthAddress.ToString();
         }
+
+        public static bool operator ==(EthAccount a, EthAccount b)
+        {
+            return a.PrivateKey == b.PrivateKey;
+        }
+
+        public static bool operator !=(EthAccount a, EthAccount b)
+        {
+            return a.PrivateKey != b.PrivateKey;
+        }
     }
 }

Framework/Utils/EthAddress.cs

@@ -6,7 +6,7 @@
     }
 
     [Serializable]
-    public class EthAddress
+    public class EthAddress : IComparable<EthAddress>
     {
         public EthAddress(string address)
         {
@@ -15,10 +15,14 @@
 
         public string Address { get; }
 
+        public int CompareTo(EthAddress? other)
+        {
+            return Address.CompareTo(other!.Address);
+        }
+
         public override bool Equals(object? obj)
         {
-            return obj is EthAddress address &&
-                   Address == address.Address;
+            return obj is EthAddress token && Address == token.Address;
         }
 
         public override int GetHashCode()
@@ -30,5 +34,15 @@
         {
             return Address;
         }
+
+        public static bool operator ==(EthAddress a, EthAddress b)
+        {
+            return a.Address == b.Address;
+        }
+
+        public static bool operator !=(EthAddress a, EthAddress b)
+        {
+            return a.Address != b.Address;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,5 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using BlockchainUtils;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Newtonsoft.Json;
 using Utils;
 
@@ -61,6 +62,11 @@ namespace CodexContractsPlugin.Marketplace
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
+
+        public override string ToString()
+        {
+            return $"SlotFilled:[host:{Host} request:{RequestId.ToHex()} slotIndex:{SlotIndex}]";
+        }
     }
 
     public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId, IHasSlotIndex

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -1,5 +1,4 @@
 using CodexClient;
-using CodexContractsPlugin.Marketplace;
 using NUnit.Framework;
 using Utils;
 
@@ -57,33 +56,6 @@ namespace CodexReleaseTests.MarketTests
             Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
         }
 
-        private TimeSpan CalculateContractFailTimespan()
-        {
-            var config = GetContracts().Deployment.Config;
-            var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
-            var periodDuration = GetPeriodDuration();
-
-            // Each host could miss 1 proof per period,
-            // so the time we should wait is period time * requiredNum of missed proofs.
-            // Except: the proof requirement has a concept of "downtime":
-            // a segment of time where proof is not required.
-            // We calculate the probability of downtime and extend the waiting
-            // timeframe by a factor, such that all hosts are highly likely to have 
-            // failed a sufficient number of proofs.
-
-            float n = requiredNumMissedProofs;
-            return periodDuration * n * GetDowntimeFactor(config);
-        }
-
-        private float GetDowntimeFactor(MarketplaceConfig config)
-        {
-            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
-            float downtime = numBlocksInDowntimeSegment;
-            float window = 256.0f;
-            var chanceOfDowntime = downtime / window;
-            return 1.0f + chanceOfDowntime + chanceOfDowntime;
-        }
-
         private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
         {
             var cid = client.UploadFile(GenerateTestFile(5.MB()));

Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs

@@ -1,18 +0,0 @@
-using NUnit.Framework;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace CodexReleaseTests.MarketTests
-{
-    public class ContractRepairedTest
-    {
-        [Test]
-        [Ignore("TODO - Test in which a host fails, but the slot is repaired")]
-        public void ContractRepaired()
-        {
-        }
-    }
-}

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -80,6 +80,29 @@ namespace CodexReleaseTests.MarketTests
             return hosts;
         }
 
+        public ICodexNode StartOneHost()
+        {
+            var host = StartCodex(s => s
+                .WithName("singlehost")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsStorageNode()
+                )
+            );
+
+            var config = GetContracts().Deployment.Config;
+            AssertTstBalance(host, StartingBalanceTST.Tst(), nameof(StartOneHost));
+            AssertEthBalance(host, StartingBalanceEth.Eth(), nameof(StartOneHost));
+
+            host.Marketplace.MakeStorageAvailable(new StorageAvailability(
+                totalSpace: HostAvailabilitySize,
+                maxDuration: HostAvailabilityMaxDuration,
+                minPricePerBytePerSecond: 1.TstWei(),
+                totalCollateral: 999999.Tst())
+            );
+            return host;
+        }
+
         public void AssertTstBalance(ICodexNode node, TestToken expectedBalance, string message)
         {
             AssertTstBalance(node.EthAddress, expectedBalance, message);
@@ -185,7 +208,7 @@ namespace CodexReleaseTests.MarketTests
             );
         }
 
-        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
+        public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts, string purchaseId)
         {
             var fills = GetOnChainSlotFills(possibleHosts);
             return fills.Where(f => f
@@ -193,7 +216,7 @@ namespace CodexReleaseTests.MarketTests
                 .ToArray();
         }
 
-        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
+        public SlotFill[] GetOnChainSlotFills(IEnumerable<ICodexNode> possibleHosts)
         {
             var events = GetContracts().GetEvents(GetTestRunTimeRange());
             var fills = events.GetSlotFilledEvents();
@@ -356,6 +379,33 @@ namespace CodexReleaseTests.MarketTests
             }, description);
         }
 
+        protected TimeSpan CalculateContractFailTimespan()
+        {
+            var config = GetContracts().Deployment.Config;
+            var requiredNumMissedProofs = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
+            var periodDuration = GetPeriodDuration();
+            var gracePeriod = periodDuration;
+
+            // Each host could miss 1 proof per period,
+            // so the time we should wait is period time * requiredNum of missed proofs.
+            // Except: the proof requirement has a concept of "downtime":
+            // a segment of time where proof is not required.
+            // We calculate the probability of downtime and extend the waiting
+            // timeframe by a factor, such that all hosts are highly likely to have 
+            // failed a sufficient number of proofs.
+
+            float n = requiredNumMissedProofs;
+            return gracePeriod + (periodDuration * n * GetDowntimeFactor(config));
+        }
+
+        private float GetDowntimeFactor(MarketplaceConfig config)
+        {
+            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
+            float downtime = numBlocksInDowntimeSegment;
+            float window = 256.0f;
+            var chanceOfDowntime = downtime / window;
+            return 1.0f + chanceOfDowntime + chanceOfDowntime;
+        }
         public class SlotFill
         {
             public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)

Tests/CodexReleaseTests/MarketTests/RepairTest.cs

@@ -0,0 +1,185 @@
+using CodexClient;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class RepairTest : MarketplaceAutoBootstrapDistTest
+    {
+        #region Setup
+
+        private readonly ByteSize Filesize;
+        private readonly uint Slots;
+        private readonly uint Tolerance;
+        private readonly ByteSize EncodedFilesize;
+        private readonly ByteSize SlotSize;
+
+        public RepairTest()
+        {
+            Filesize = 32.MB();
+            Slots = 4;
+            Tolerance = 2;
+
+            EncodedFilesize = new ByteSize(Filesize.SizeInBytes * (Slots / Tolerance));
+            SlotSize = new ByteSize(EncodedFilesize.SizeInBytes / Slots);
+            Assert.That(IsPowerOfTwo(SlotSize));
+            Assert.That(Slots, Is.LessThan(NumberOfHosts));
+        }
+
+        protected override int NumberOfHosts => 5;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => SlotSize.Multiply(1.1); // Each host can hold 1 slot.
+        protected override TimeSpan HostAvailabilityMaxDuration => GetPeriodDuration() * 100;
+
+        private static bool IsPowerOfTwo(ByteSize size)
+        {
+            var x = size.SizeInBytes;
+            return (x != 0) && ((x & (x - 1)) == 0);
+        }
+
+        #endregion
+
+        [Test]
+        [Combinatorial]
+        public void RollingRepairSingleFailure(
+            [Values(10)] int numFailures)
+        {
+            var hosts = StartHosts().ToList();
+            var client = StartClients().Single();
+
+            var contract = CreateStorageRequest(client);
+            contract.WaitForStorageContractStarted();
+            // All slots are filled.
+
+            for (var i = 0; i < numFailures; i++)
+            {
+                Log($"Failure step: {i}");
+
+                // Start a new host. Add it to the back of the list:
+                hosts.Add(StartOneHost());
+
+                var fill = GetSlotFillByOldestHost(hosts);
+
+                Log($"Causing failure for host: {fill.Host.GetName()} slotIndex: {fill.SlotFilledEvent.SlotIndex}");
+                hosts.Remove(fill.Host);
+                fill.Host.Stop(waitTillStopped: true);
+
+                // The slot should become free.
+                WaitForSlotFreedEvent(contract, fill.SlotFilledEvent.SlotIndex);
+
+                // One of the other hosts should pick up the free slot.
+                WaitForNewSlotFilledEvent(contract, fill.SlotFilledEvent.SlotIndex);
+            }
+        }
+
+        private void WaitForSlotFreedEvent(IStoragePurchaseContract contract, ulong slotIndex)
+        {
+            Log(nameof(WaitForSlotFreedEvent));
+            var start = DateTime.UtcNow;
+            var timeout = CalculateContractFailTimespan();
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var slotsFreed = events.GetSlotFreedEvents();
+                Log($"Slots freed this period: {slotsFreed.Length}");
+
+                foreach (var free in slotsFreed)
+                {
+                    if (free.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant())
+                    {
+                        if (free.SlotIndex == slotIndex)
+                        {
+                            Log("Found the correct slotFree event");
+                            return;
+                        }
+                    }
+                }
+
+                GetContracts().WaitUntilNextPeriod();
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private void WaitForNewSlotFilledEvent(IStoragePurchaseContract contract, ulong slotIndex)
+        {
+            Log(nameof(WaitForNewSlotFilledEvent));
+            var start = DateTime.UtcNow;
+            var timeout = contract.Purchase.Expiry;
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var newTimeRange = new TimeRange(start, DateTime.UtcNow); // We only want to see new fill events.
+                var events = GetContracts().GetEvents(newTimeRange);
+                var slotFillEvents = events.GetSlotFilledEvents();
+
+                var matches = slotFillEvents.Where(f =>
+                {
+                    return
+                        f.RequestId.ToHex().ToLowerInvariant() == contract.PurchaseId.ToLowerInvariant() &&
+                        f.SlotIndex == slotIndex;
+                }).ToArray();
+
+                if (matches.Length > 1)
+                {
+                    var msg = string.Join(",", matches.Select(f => f.ToString()));
+                    Assert.Fail($"Somehow, the slot got filled multiple times: {msg}");
+                }
+                if (matches.Length == 1)
+                {
+                    Log($"Found the correct new slotFilled event: {matches[0].ToString()}");
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(15));
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvent)} for contract {contract.PurchaseId} and slotIndex {slotIndex} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private SlotFill GetSlotFillByOldestHost(List<ICodexNode> hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var copy = hosts.ToArray();
+            foreach (var host in copy)
+            {
+                var fill = GetFillByHost(host, fills);
+                if (fill == null)
+                {
+                    // This host didn't fill anything.
+                    // Move this one to the back of the list.
+                    hosts.Remove(host);
+                    hosts.Add(host);
+                }
+                else
+                {
+                    return fill;
+                }
+            }
+            throw new Exception("None of the hosts seem to have filled a slot.");
+        }
+
+        private SlotFill? GetFillByHost(ICodexNode host, SlotFill[] fills)
+        {
+            // If these is more than 1 fill by this host, the test is misconfigured.
+            // The availability size of the host should guarantee it can fill 1 slot maximum.
+            return fills.SingleOrDefault(f => f.Host.EthAddress == host.EthAddress);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(Filesize));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = TimeSpan.FromDays(2.0),
+                Expiry = TimeSpan.FromMinutes(10.0),
+                MinRequiredNumberOfNodes = Slots,
+                NodeFailureTolerance = Tolerance,
+                PricePerBytePerSecond = 10.TstWei(),
+                ProofProbability = 20,
+                CollateralPerByte = 1.TstWei()
+            });
+        }
+    }
+}

Tests/FrameworkTests/Utils/EthAccountEqualityTests.cs

@@ -0,0 +1,31 @@
+using GethPlugin;
+using NUnit.Framework;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class EthAccountEqualityTests
+    {
+        [Test]
+        public void Accounts()
+        {
+            var account1 = EthAccountGenerator.GenerateNew();
+            var account2 = EthAccountGenerator.GenerateNew();
+
+            Assert.That(account1, Is.EqualTo(account1));
+            Assert.That(account1 == account1);
+            Assert.That(account1 != account2);
+        }
+
+        [Test]
+        public void Addresses()
+        {
+            var address1 = EthAccountGenerator.GenerateNew().EthAddress;
+            var address2 = EthAccountGenerator.GenerateNew().EthAddress;
+
+            Assert.That(address1, Is.EqualTo(address1));
+            Assert.That(address1 == address1);
+            Assert.That(address1 != address2);
+        }
+    }
+}