diff --git a/Framework/KubernetesWorkflow/LogHandler.cs b/Framework/KubernetesWorkflow/LogHandler.cs
index af77ef75..b8a8fa4b 100644
--- a/Framework/KubernetesWorkflow/LogHandler.cs
+++ b/Framework/KubernetesWorkflow/LogHandler.cs
@@ -40,6 +40,10 @@ namespace KubernetesWorkflow
protected override void ProcessLine(string line)
{
+ // This line is not useful and has no topic so we can't filter it with
+ // normal log-level controls.
+ if (line.Contains("Received JSON-RPC response") && !line.Contains("topics=")) return;
+
LogFile.WriteRaw(line);
}
}
diff --git a/Framework/Utils/EthAddress.cs b/Framework/Utils/EthAddress.cs
index 61c2776c..a16f4b46 100644
--- a/Framework/Utils/EthAddress.cs
+++ b/Framework/Utils/EthAddress.cs
@@ -30,5 +30,15 @@
{
return Address;
}
+
+ public static bool operator ==(EthAddress a, EthAddress b)
+ {
+ return a.Address == b.Address;
+ }
+
+ public static bool operator !=(EthAddress a, EthAddress b)
+ {
+ return a.Address != b.Address;
+ }
}
}
diff --git a/ProjectPlugins/CodexClient/CodexAccess.cs b/ProjectPlugins/CodexClient/CodexAccess.cs
index ca11ebee..e9a49224 100644
--- a/ProjectPlugins/CodexClient/CodexAccess.cs
+++ b/ProjectPlugins/CodexClient/CodexAccess.cs
@@ -11,7 +11,7 @@ namespace CodexClient
private readonly ILog log;
private readonly IHttpFactory httpFactory;
private readonly IProcessControl processControl;
- private ICodexInstance instance;
+ private readonly ICodexInstance instance;
private readonly Mapper mapper = new Mapper();
public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
@@ -25,8 +25,6 @@ namespace CodexClient
public void Stop(bool waitTillStopped)
{
processControl.Stop(waitTillStopped);
- // Prevents accidental use after stop:
- instance = null!;
}
public IDownloadedLog DownloadLog(string additionalName = "")
diff --git a/ProjectPlugins/CodexClient/CodexClient.csproj b/ProjectPlugins/CodexClient/CodexClient.csproj
index cc417a9e..17a76836 100644
--- a/ProjectPlugins/CodexClient/CodexClient.csproj
+++ b/ProjectPlugins/CodexClient/CodexClient.csproj
@@ -30,6 +30,7 @@
+
diff --git a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs
index 618ca1c3..a1ae1394 100644
--- a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs
+++ b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs
@@ -1,4 +1,5 @@
using CodexClient.Hooks;
+using CodexContractsPlugin.Marketplace;
using Logging;
using Newtonsoft.Json;
using Utils;
@@ -14,7 +15,7 @@ namespace CodexClient
void WaitForStorageContractSubmitted();
void WaitForStorageContractStarted();
void WaitForStorageContractFinished();
- void WaitForContractFailed();
+ void WaitForContractFailed(MarketplaceConfig config);
}
public class StoragePurchaseContract : IStoragePurchaseContract
@@ -99,7 +100,7 @@ namespace CodexClient
AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
}
- public void WaitForContractFailed()
+ public void WaitForContractFailed(MarketplaceConfig config)
{
if (!contractStartedUtc.HasValue)
{
@@ -107,9 +108,27 @@ namespace CodexClient
}
var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+ var minTimeout = TimeNeededToFailEnoughProofsToFreeASlot(config);
+
+ if (timeout < minTimeout)
+ {
+ throw new ArgumentOutOfRangeException(
+ $"Test is misconfigured. Assuming a proof is required every period, it will take {Time.FormatDuration(minTimeout)} " +
+ $"to fail enough proofs for a slot to be freed. But, the storage contract will complete in {Time.FormatDuration(timeout)}. " +
+ $"Increase the duration."
+ );
+ }
+
WaitForStorageContractState(timeout, "failed");
}
+ private TimeSpan TimeNeededToFailEnoughProofsToFreeASlot(MarketplaceConfig config)
+ {
+ var numMissedProofsRequiredForFree = config.Collateral.MaxNumberOfSlashes;
+ var timePerProof = TimeSpan.FromSeconds(config.Proofs.Period);
+ return timePerProof * (numMissedProofsRequiredForFree + 1);
+ }
+
private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
{
var waitStart = DateTime.UtcNow;
diff --git a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs
index 455306be..5c1d3be3 100644
--- a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs
+++ b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs
@@ -1,6 +1,9 @@
using CodexClient;
+using CodexContractsPlugin.ChainMonitor;
using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
using NUnit.Framework;
+using System.Numerics;
using Utils;
namespace CodexReleaseTests.MarketTests
@@ -21,7 +24,7 @@ namespace CodexReleaseTests.MarketTests
{
var hosts = StartHosts();
var client = StartClients().Single();
- StartValidator();
+ var validator = StartValidator();
var request = CreateStorageRequest(client);
@@ -32,32 +35,97 @@ namespace CodexReleaseTests.MarketTests
AssertContractSlotsAreFilledByHosts(request, hosts);
hosts.Stop(waitTillStopped: true);
+
+ var config = GetContracts().Deployment.Config;
+ request.WaitForContractFailed(config);
- WaitForSlotFreedEvents();
- AssertProofMissedReports();
+ var frees = GetOnChainSlotFrees(hosts);
+ Assert.That(frees.Length, Is.EqualTo(
+ request.Purchase.MinRequiredNumberOfNodes - request.Purchase.NodeFailureTolerance));
- request.WaitForContractFailed();
+ var periodReports = GetPeriodMonitorReports();
+ var missedProofs = periodReports.Reports.SelectMany(r => r.MissedProofs).ToArray();
+ AssertEnoughProofsWereMissedForSlotFree(frees, missedProofs, config);
+
+ AssertClientPaidNothing(client);
+ AssertValidatorWasPaidPerMissedProof(validator, request, missedProofs, config);
+ AssertHostCollateralWasBurned(hosts, request);
}
- private void WaitForSlotFreedEvents()
+ private void AssertClientPaidNothing(ICodexNode client)
{
- Log(nameof(WaitForSlotFreedEvents));
+ AssertTstBalance(client, StartingBalanceTST.Tst(), "Client should not have paid for failed contract.");
+ }
- var start = DateTime.UtcNow;
- var timeout = CalculateContractFailTimespan();
+ private void AssertValidatorWasPaidPerMissedProof(ICodexNode validator, IStoragePurchaseContract request, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+ {
+ var rewardPerMissedProof = GetValidatorRewardPerMissedProof(request, config);
+ var totalValidatorReward = rewardPerMissedProof * missedProofs.Length;
- while (DateTime.UtcNow < start + timeout)
+ AssertTstBalance(validator, StartingBalanceTST.Tst() + totalValidatorReward, $"Validator is rewarded per slot marked as missing. " +
+ $"numberOfMissedProofs: {missedProofs.Length} rewardPerMissedProof: {rewardPerMissedProof}");
+ }
+
+ private TestToken GetCollatoralPerSlot(IStoragePurchaseContract request)
+ {
+ var slotSize = new ByteSize(Convert.ToInt64(request.GetStatus()!.Request.Ask.SlotSize));
+ return new TestToken(request.Purchase.CollateralPerByte.TstWei * slotSize.SizeInBytes);
+ }
+
+ private void AssertHostCollateralWasBurned(ICodexNodeGroup hosts, IStoragePurchaseContract request)
+ {
+ var slotFills = GetOnChainSlotFills(hosts);
+ foreach (var host in hosts)
{
- var events = GetContracts().GetEvents(GetTestRunTimeRange());
- var slotFreed = events.GetSlotFreedEvents();
- if (slotFreed.Length == NumberOfSlots)
- {
- Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
- return;
- }
- GetContracts().WaitUntilNextPeriod();
+ AssertHostCollateralWasBurned(host, slotFills, request);
}
- Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
+ }
+
+ private void AssertHostCollateralWasBurned(ICodexNode host, SlotFill[] slotFills, IStoragePurchaseContract request)
+ {
+ // In case of a failed contract, the entire slotColateral is lost.
+ var filledByHost = slotFills.Where(f => f.Host.EthAddress == host.EthAddress).ToArray();
+ var numSlotsOfHost = filledByHost.Length;
+ var collatoralPerSlot = GetCollatoralPerSlot(request);
+ var totalCost = collatoralPerSlot * numSlotsOfHost;
+
+ AssertTstBalance(host, StartingBalanceTST.Tst() - totalCost, $"Host has lost collateral for each slot. " +
+ $"numberOfSlotsByHost: {numSlotsOfHost} collateralPerSlot: {collatoralPerSlot}");
+ }
+
+ private TestToken GetValidatorRewardPerMissedProof(IStoragePurchaseContract request, MarketplaceConfig config)
+ {
+ var collatoralPerSlot = GetCollatoralPerSlot(request);
+ var slashPercentage = config.Collateral.SlashPercentage;
+ var validatorRewardPercentage = config.Collateral.ValidatorRewardPercentage;
+
+ var rewardPerMissedProof =
+ PercentageOf(
+ PercentageOf(collatoralPerSlot, slashPercentage),
+ validatorRewardPercentage);
+
+ return rewardPerMissedProof;
+ }
+
+ private TestToken PercentageOf(TestToken value, byte percentage)
+ {
+ var p = new BigInteger(percentage);
+ return new TestToken((value.TstWei * p) / 100);
+ }
+
+ private void AssertEnoughProofsWereMissedForSlotFree(SlotFree[] frees, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+ {
+ foreach (var free in frees)
+ {
+ AssertEnoughProofsWereMissedForSlotFree(free, missedProofs, config);
+ }
+ }
+
+ private void AssertEnoughProofsWereMissedForSlotFree(SlotFree free, PeriodProofMissed[] missedProofs, MarketplaceConfig config)
+ {
+ var missedByHost = missedProofs.Where(p => p.Host != null && p.Host.Address == free.Host.EthAddress.Address).ToArray();
+ var maxNumMissedProofsBeforeFreeSlot = config.Collateral.MaxNumberOfSlashes;
+ Assert.That(missedByHost.Length, Is.EqualTo(maxNumMissedProofsBeforeFreeSlot));
}
private TimeSpan CalculateContractFailTimespan()
@@ -92,8 +160,8 @@ namespace CodexReleaseTests.MarketTests
var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
{
- Duration = TimeSpan.FromHours(1.0),
- Expiry = TimeSpan.FromHours(0.2),
+ Duration = TimeSpan.FromMinutes(10.0),
+ Expiry = TimeSpan.FromMinutes(5.0),
MinRequiredNumberOfNodes = NumberOfSlots,
NodeFailureTolerance = 1,
PricePerBytePerSecond = pricePerBytePerSecond,
diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
index 17d83e3a..04957541 100644
--- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
+++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs
@@ -68,8 +68,8 @@ namespace CodexReleaseTests.MarketTests
var hosts = StartCodex(NumberOfHosts, s => s
.WithName("host")
.WithBlockTTL(HostBlockTTL)
- .WithBlockMaintenanceNumber(100000)
- .WithBlockMaintenanceInterval(HostBlockTTL / 4)
+ .WithBlockMaintenanceNumber(100)
+ .WithBlockMaintenanceInterval(HostBlockTTL / 2)
.EnableMarketplace(GetGeth(), GetContracts(), m => m
.WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
.AsStorageNode()
@@ -236,6 +236,29 @@ namespace CodexReleaseTests.MarketTests
}).ToArray();
}
+ public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts, string purchaseId)
+ {
+ var fills = GetOnChainSlotFrees(possibleHosts);
+ return fills.Where(f => f
+ .SlotFreedEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
+ .ToArray();
+ }
+
+ public SlotFree[] GetOnChainSlotFrees(ICodexNodeGroup possibleHosts)
+ {
+ var events = GetContracts().GetEvents(GetTestRunTimeRange());
+ var fills = GetOnChainSlotFills(possibleHosts);
+ var frees = events.GetSlotFreedEvents();
+ return frees.Select(f =>
+ {
+ var matchingFill = fills.Single(fill => fill.SlotFilledEvent.RequestId == f.RequestId &&
+ fill.SlotFilledEvent.SlotIndex == f.SlotIndex);
+
+ return new SlotFree(f, matchingFill.Host);
+
+ }).ToArray();
+ }
+
protected void AssertClientHasPaidForContract(TestToken pricePerBytePerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
{
var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts);
@@ -359,13 +382,6 @@ namespace CodexReleaseTests.MarketTests
}, description);
}
- protected void AssertEnoughProofMissedForSlotFree(ICodexNodeGroup hosts)
- {
- var slotFills = GetOnChainSlotFills(hosts);
-
- todo for each filled slot, there should be enough proofs missed to trigger the slot-free event.
- }
-
public class SlotFill
{
public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
@@ -378,6 +394,18 @@ namespace CodexReleaseTests.MarketTests
public ICodexNode Host { get; }
}
+ public class SlotFree
+ {
+ public SlotFree(SlotFreedEventDTO slotFreedEvent, ICodexNode host)
+ {
+ SlotFreedEvent = slotFreedEvent;
+ Host = host;
+ }
+
+ public SlotFreedEventDTO SlotFreedEvent { get; }
+ public ICodexNode Host { get; }
+ }
+
private class MarketplaceHandle
{
public MarketplaceHandle(IGethNode geth, ICodexContracts contracts, ChainMonitor monitor)
diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs
index cd0a4299..a1b26c73 100644
--- a/Tests/CodexReleaseTests/Parallelism.cs
+++ b/Tests/CodexReleaseTests/Parallelism.cs
@@ -1,6 +1,6 @@
using NUnit.Framework;
-[assembly: LevelOfParallelism(10)]
+[assembly: LevelOfParallelism(1)]
namespace CodexReleaseTests.DataTests
{
}
diff --git a/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs b/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs
new file mode 100644
index 00000000..72cb8a67
--- /dev/null
+++ b/Tests/FrameworkTests/Utils/EthAddressEqualityTests.cs
@@ -0,0 +1,46 @@
+using GethPlugin;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+ [TestFixture]
+ public class EthAddressEqualityTests
+ {
+ [Test]
+ [Combinatorial]
+ public void Equal(
+ [Values(1, 2, 3, 4, 5)] int runs
+ )
+ {
+ var account = EthAccountGenerator.GenerateNew();
+
+ var str = account.EthAddress.Address;
+
+ var addr = new EthAddress(str);
+
+ Assert.That(addr, Is.EqualTo(account.EthAddress));
+ Assert.That(addr == account.EthAddress);
+ Assert.That(!(addr != account.EthAddress));
+ }
+
+ [Test]
+ [Combinatorial]
+ public void NotEqual(
+ [Values(1, 2, 3, 4, 5)] int runs
+ )
+ {
+ var account1 = EthAccountGenerator.GenerateNew();
+ var account2 = EthAccountGenerator.GenerateNew();
+
+ Assert.That(account1.EthAddress, Is.Not.EqualTo(account2.EthAddress));
+ Assert.That(account1.EthAddress != account2.EthAddress);
+ Assert.That(!(account1.EthAddress == account2.EthAddress));
+ }
+ }
+}