test: testing slot expiry

The test is mostly working, except that the node needs to be updated to call the onReservationAdded callback once availability marked unused

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,8 +5,7 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-7efa917";
-
+        public const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
@@ -31,7 +30,7 @@ namespace DistTestCore.Codex
 
             AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
             AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
-            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
             // This makes the node announce itself to its local (pod) IP address.
             AddEnvVar("NAT_IP_AUTO", "true");
@@ -58,6 +57,14 @@ namespace DistTestCore.Codex
                 AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
             }
 
+			if (config.SimulateProofFailures != null)
+            {
+                AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+			// if (config.EnableValidator == true)
+            // {
+            //     AddEnvVar("CODEX_VALIDATOR", "true");
+            // }
             if (config.MarketplaceConfig != null)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
@@ -75,8 +82,15 @@ namespace DistTestCore.Codex
 
                 if (config.MarketplaceConfig.IsValidator)
                 {
-                    AddEnvVar("CODEX_VALIDATOR", "true");
+                   AddEnvVar("CODEX_VALIDATOR", "true");
                 }
+            }
+			// if (config.MarketplaceConfig != null) {
+            //     AddEnvVar("CODEX_PERSISTENCE", "true");
+            // }
+
+            if(!string.IsNullOrEmpty(config.NameOverride)) {
+                AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
         }
 

DistTestCore/Codex/CodexStartupConfig.cs

@@ -10,13 +10,26 @@ namespace DistTestCore.Codex
             LogLevel = logLevel;
         }
 
+        public string LogLevelWithTopics()
+        {
+            var level = LogLevel.ToString()!.ToUpperInvariant();
+            if (LogTopics != null && LogTopics.Count() > 0)
+            {
+                level = $"INFO;{level}: {string.Join(",", LogTopics.Where(s => !string.IsNullOrEmpty(s)))}";
+            }
+            return level;
+        }
+
         public string? NameOverride { get; set; }
         public Location Location { get; set; }
-        public CodexLogLevel LogLevel { get; }
+        public CodexLogLevel LogLevel { get; set; }
+        public string[]? LogTopics { get; set; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public uint? SimulateProofFailures { get; set; }
+        public bool? EnableValidator { get; set; }
     }
 }

DistTestCore/CodexSetup.cs

@@ -8,6 +8,12 @@ namespace DistTestCore
     {
         ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
+        ICodexSetup WithLogLevel(CodexLogLevel level);
+        /// <summary>
+        /// Sets the log level for codex. The default level is INFO and the
+        /// log level is applied only to the supplied topics.
+        /// </summary>
+        ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup WithBlockTTL(TimeSpan duration);
@@ -15,8 +21,16 @@ namespace DistTestCore
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
+		/// <summary>
+        /// Provides an invalid proof every N proofs
+        /// </summary>
+        ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
+        /// <summary>
+        /// Enables the validation module in the node
+        /// </summary>
+        // ICodexSetup WithValidator();
     }
-    
+
     public class CodexSetup : CodexStartupConfig, ICodexSetup
     {
         public int NumberOfNodes { get; }
@@ -45,6 +59,19 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithLogLevel(CodexLogLevel level)
+        {
+            LogLevel = level;
+            return this;
+        }
+
+        public ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics)
+        {
+            LogLevel = level;
+            LogTopics = topics;
+            return this;
+        }
+
         public ICodexSetup WithStorageQuota(ByteSize storageQuota)
         {
             StorageQuota = storageQuota;
@@ -70,7 +97,7 @@ namespace DistTestCore
 
         public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
         {
-            return EnableMarketplace(initialBalance, initialEther, false);
+			return EnableMarketplace(initialBalance, initialEther, false);
         }
 
         public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
@@ -79,6 +106,18 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithSimulateProofFailures(uint failEveryNProofs)
+        {
+            SimulateProofFailures = failEveryNProofs;
+            return this;
+        }
+
+        // public ICodexSetup WithValidator()
+        // {
+        //     EnableValidator = true;
+        //     return this;
+        // }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
@@ -87,9 +126,11 @@ namespace DistTestCore
 
         private IEnumerable<string> DescribeArgs()
         {
-            yield return $"LogLevel={LogLevel}";
+            yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
+            if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";
+            if (SimulateProofFailures != null) yield return $"SimulateProofFailures={SimulateProofFailures}";
+            if (MarketplaceConfig != null) yield return $"IsValidator={MarketplaceConfig.IsValidator}";
         }
     }
 }

DistTestCore/CodexStarter.cs

@@ -20,6 +20,7 @@ namespace DistTestCore
             LogSeparator();
             LogStart($"Starting {codexSetup.Describe()}...");
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
+            gethStartResult = lifecycle.GethStarter.BringOnlineValidatorFor(codexSetup, gethStartResult);
 
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
 

DistTestCore/DistTestCore.csproj

@@ -5,10 +5,6 @@
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
-    <IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(IsArm64)'=='true'">
-    <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>

DistTestCore/GethStarter.cs

@@ -30,6 +30,17 @@ namespace DistTestCore
             return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
+        public GethStartResult BringOnlineValidatorFor(CodexSetup codexSetup, GethStartResult previousResult)
+        {
+            // allow marketplace and validator to be enabled on the same Codex node
+            if (previousResult.CompanionNode != null || (codexSetup.EnableValidator ?? false) == false) return previousResult;
+
+            var marketplaceNetwork = marketplaceNetworkCache.Get();
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
+
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
+        }
+
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
             if (marketplaceConfig.InitialTestTokens.Amount == 0) return;

DistTestCore/Helpers/AssertHelpers.cs

@@ -10,8 +10,7 @@ namespace DistTestCore.Helpers
         {
             try
             {
-                var c = constraint.Resolve();
-                Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
+                Time.WaitUntil(() => constraint.Resolve().ApplyTo(actual()).IsSuccess);
             }
             catch (TimeoutException)
             {

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -8,7 +8,7 @@ namespace DistTestCore.Marketplace
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
         public override string AppName => "codex-contracts";
-        public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -11,7 +11,7 @@ namespace DistTestCore.Marketplace
         public const string AccountsFilename = "accounts.csv";
 
         public override string AppName => "geth";
-        public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -5,6 +5,7 @@ using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using System.Numerics;
+using System.Linq;
 using Utils;
 
 namespace DistTestCore.Marketplace
@@ -12,7 +13,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -32,17 +33,18 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null)
         {
+            var expiryTimestamp = expiry is null ? null : ((DateTimeOffset)expiry).ToUnixTimeSeconds().ToString();
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToDecInt(duration.TotalSeconds),
                 proofProbability = ToDecInt(proofProbability),
                 reward = ToDecInt(pricePerSlotPerSecond),
                 collateral = ToDecInt(requiredCollateral),
-                expiry = null,
+                expiry = expiryTimestamp,
                 nodes = minRequiredNumberOfNodes,
-                tolerance = null,
+                tolerance = tolerance,
             };
 
             Log($"Requesting storage for: {contentId.Id}... (" +
@@ -50,7 +52,9 @@ namespace DistTestCore.Marketplace
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +
-                $"duration: {Time.FormatDuration(duration)})");
+                $"duration: {Time.FormatDuration(duration)}, " +
+                $"expiry: {expiry:yyyy-MM-dd HH:mm:ss}, " +
+                $"tolerance: {tolerance})");
 
             var response = codexAccess.RequestStorage(request, contentId.Id);
 
@@ -123,7 +127,7 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null)
         {
             Unavailable();
             return null!;
@@ -187,6 +191,29 @@ namespace DistTestCore.Marketplace
             WaitForStorageContractState(timeout, "finished");
         }
 
+        /// <summary>
+        /// Wait for contract to terminate. Which means that it reaches one of the following states: Finished, Cancelled, Failed
+        /// </summary>
+        public void WaitForStorageContractTerminated()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var gracePeriod = TimeSpan.FromSeconds(10);
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            string[] terminatedStates = { "finished", "failed", "cancelled" };
+            WaitForStorageContractState(timeout, terminatedStates);
+        }
+
+        public void WaitForStorageContractFailed(TimeSpan timeout)
+        {
+            WaitForStorageContractState(timeout, "failed");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+
         /// <summary>
         /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
         /// </summary>
@@ -205,12 +232,18 @@ namespace DistTestCore.Marketplace
         }
 
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
+        {
+            string[] states = { desiredState };
+            WaitForStorageContractState(timeout, states);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string[] desiredState)
         {
             var lastState = "";
             var waitStart = DateTime.UtcNow;
-            
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
+
+            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{string.Join("/", desiredState)}'.");
+            while (!desiredState.Contains(lastState))
             {
                 var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
                 var statusJson = JsonConvert.SerializeObject(purchaseStatus);
@@ -229,10 +262,10 @@ namespace DistTestCore.Marketplace
 
                 if (DateTime.UtcNow - waitStart > timeout)
                 {
-                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                    Assert.Fail($"Contract did not reach '{string.Join("/", desiredState)}' within timeout. {statusJson}");
                 }
             }
-            log.Log($"Contract '{desiredState}'.");
+            log.Log($"Contract '{string.Join("/", desiredState)}'.");
         }
 
         public CodexStoragePurchase GetPurchaseStatus(string purchaseId)

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -5,7 +5,7 @@ namespace DistTestCore.Metrics
     public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "prometheus";
-        public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd";
+        public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Tokens.cs

@@ -58,6 +58,16 @@
         {
             return $"{Amount} TestTokens";
         }
+
+        public static TestToken operator - (TestToken t1, int amount)
+        {
+            return t1 - Convert.ToDecimal(amount);
+        }
+
+        public static TestToken operator - (TestToken t1, decimal amount)
+        {
+            return (t1.Amount - amount).TestTokens();
+        }
     }
 
     public static class TokensIntExtensions

Tests/BasicTests/ExampleTests.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using NUnit.Framework;
 
 namespace Tests.BasicTests
@@ -46,6 +47,7 @@ namespace Tests.BasicTests
             var fileSize = 10.MB();
 
             var seller = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
                             .WithStorageQuota(11.GB())
                             .EnableMarketplace(sellerInitialBalance));
 
@@ -59,11 +61,12 @@ namespace Tests.BasicTests
             var testFile = GenerateTestFile(fileSize);
 
             var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
 
             buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
-            
+
             var contentId = buyer.UploadFile(testFile);
             var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
                 pricePerSlotPerSecond: 2.TestTokens(),

Tests/Marketplace/Interactions.cs

@@ -0,0 +1,87 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class SlotSelection : AutoBootstrapDistTest
+    {
+
+        [Test]
+        public void RequestExpiresIfNotFilledAndMoneyAreReturned()
+        {
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+            var fileSize = 10.MB();
+
+            var seller1 = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                            .WithStorageQuota(50.MB())
+                            .EnableMarketplace(sellerInitialBalance)
+                            .WithName("seller1"));
+
+            seller1.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+
+            // The two availabilities are needed until https://github.com/codex-storage/nim-codex/pull/535 is merged
+            seller1.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+            seller1.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var seller2 = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                            .WithStorageQuota(50.MB())
+                            .EnableMarketplace(sellerInitialBalance)
+                            .WithName("seller2"));
+
+            seller2.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            seller2.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+            seller2.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var testFile = GenerateTestFile(fileSize);
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
+                .EnableMarketplace(buyerInitialBalance)
+                .WithName("buyer"));
+
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
+            var contentId = buyer.UploadFile(testFile);
+            var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
+                pricePerSlotPerSecond: 2.TestTokens(),
+                requiredCollateral: 10.TestTokens(),
+                minRequiredNumberOfNodes: 3,
+                proofProbability: 5,
+                duration: TimeSpan.FromMinutes(1),
+                expiry: DateTime.Now.AddMinutes(2));
+
+            Time.Sleep(TimeSpan.FromSeconds(100));
+
+            seller1.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+            seller2.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+            buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
+
+            purchaseContract.WaitForStorageContractFailed(TimeSpan.FromSeconds(120));
+
+            seller1.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance), "Seller was not returned collateral.");
+            seller2.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance), "Seller was not returned collateral.");
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance), "Buyer was not returned money for the request.");
+        }
+    }
+}

Tests/Marketplace/Proofs.cs

@@ -0,0 +1,97 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class MarketplaceTests : DistTest
+    {
+        [Test]
+        public void HostThatMissesProofsIsPaidOutLessThanHostThatDoesNotMissProofs()
+        {
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+
+            var seller = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                .WithStorageQuota(21.GB())
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller"));
+
+            var sellerWithFailures = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                .WithStorageQuota(21.GB())
+                .WithBootstrapNode(seller)
+                .WithSimulateProofFailures(2)
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller with failures"));
+
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
+                .WithBootstrapNode(seller)
+                .EnableMarketplace(buyerInitialBalance)
+                .WithName("buyer"));
+
+            var validator = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "validator")
+                .WithBootstrapNode(seller)
+                // .WithValidator()
+                .EnableMarketplace(0.TestTokens(), 2.Eth(), true)
+                .WithName("validator"));
+
+            seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
+            seller.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            sellerWithFailures.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            seller.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            sellerWithFailures.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var fileSize = 10.MB();
+            var testFile = GenerateTestFile(fileSize);
+            var contentId = buyer.UploadFile(testFile);
+
+            var purchaseContract = buyer.Marketplace.RequestStorage(
+                contentId,
+                pricePerSlotPerSecond: 2.TestTokens(),
+                requiredCollateral: 10.TestTokens(),
+                minRequiredNumberOfNodes: 2,
+                proofProbability: 2,
+                duration: TimeSpan.FromMinutes(3));
+
+            // Time.Sleep(TimeSpan.FromMinutes(1));
+
+            // seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+
+            purchaseContract.WaitForStorageContractStarted(fileSize);
+            purchaseContract.WaitForStorageContractFinished();
+
+            var sellerBalance = seller.Marketplace.GetBalance();
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.LessThan(sellerBalance), "Seller that was slashed should have less balance than seller that was not slashed.");
+
+            new List<IOnlineCodexNode>(){seller, sellerWithFailures, buyer, validator}.ForEach(node => node.DownloadLog());
+        }
+    }
+}