Add validator and simulation of proof failures

- add validator to codex node setup, and start companion node for it
- add simulate-proof-failures to codex node setup
- allow log-level to log topics at a specific level

# Conflicts:
#	DistTestCore/Codex/CodexContainerRecipe.cs
#	DistTestCore/Codex/CodexStartupConfig.cs
#	DistTestCore/CodexSetup.cs

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,8 +5,7 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-7efa917";
+        public const string DockerImage = "codexstorage/nim-codex:sha-1d161d3";
-
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
@@ -31,7 +30,12 @@ namespace DistTestCore.Codex
 
             AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
             AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
-            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+
+            var level = config.LogLevel.ToString()!.ToUpperInvariant();
+            if (config.LogTopics != null && config.LogTopics.Count() > 0){
+                level = $"INFO;{level}: {string.Join(",", config.LogTopics.Where(s => !string.IsNullOrEmpty(s)))}";
+            }
+            AddEnvVar("CODEX_LOG_LEVEL", level);
 
             // This makes the node announce itself to its local (pod) IP address.
             AddEnvVar("NAT_IP_AUTO", "true");
@@ -58,7 +62,15 @@ namespace DistTestCore.Codex
                 AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
             }
 
-            if (config.MarketplaceConfig != null)
+			if (config.SimulateProofFailures != null)
+            {
+                AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+			if (config.EnableValidator == true)
+            {
+                AddEnvVar("CODEX_VALIDATOR", "true");
+            }
+            if (config.MarketplaceConfig != null || config.EnableValidator == true)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
                 var companionNode = gethConfig.CompanionNode;
@@ -73,10 +85,17 @@ namespace DistTestCore.Codex
                 AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
                 AddEnvVar("CODEX_PERSISTENCE", "true");
 
-                if (config.MarketplaceConfig.IsValidator)
+                //if (config.MarketplaceConfig.IsValidator)
-                {
+                //{
-                    AddEnvVar("CODEX_VALIDATOR", "true");
+                //    AddEnvVar("CODEX_VALIDATOR", "true");
+                //}
             }
+			if (config.MarketplaceConfig != null) {
+                AddEnvVar("CODEX_PERSISTENCE", "true");
+            }
+
+            if(!string.IsNullOrEmpty(config.NameOverride)) {
+                AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
         }
 

DistTestCore/Codex/CodexStartupConfig.cs

@@ -12,11 +12,13 @@ namespace DistTestCore.Codex
 
         public string? NameOverride { get; set; }
         public Location Location { get; set; }
-        public CodexLogLevel LogLevel { get; }
+        public CodexLogLevel LogLevel { get; set; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public uint? SimulateProofFailures { get; set; }
+        public bool? EnableValidator { get; set; }
     }
 }

DistTestCore/CodexSetup.cs

@@ -15,6 +15,14 @@ namespace DistTestCore
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
+		/// <summary>
+        /// Provides an invalid proof every N proofs
+        /// </summary>
+        ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
+        /// <summary>
+        /// Enables the validation module in the node
+        /// </summary>
+        ICodexSetup WithValidator();
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -79,6 +87,18 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithSimulateProofFailures(uint failEveryNProofs)
+        {
+            SimulateProofFailures = failEveryNProofs;
+            return this;
+        }
+
+        public ICodexSetup WithValidator()
+        {
+            EnableValidator = true;
+            return this;
+        }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
@@ -89,7 +109,9 @@ namespace DistTestCore
         {
             yield return $"LogLevel={LogLevel}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
+            if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";
+            if (SimulateProofFailures != null) yield return $"SimulateProofFailures={SimulateProofFailures}";
+            if (EnableValidator != null) yield return $"EnableValidator={EnableValidator}";
         }
     }
 }

DistTestCore/CodexStarter.cs

@@ -20,6 +20,7 @@ namespace DistTestCore
             LogSeparator();
             LogStart($"Starting {codexSetup.Describe()}...");
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
+            gethStartResult = lifecycle.GethStarter.BringOnlineValidatorFor(codexSetup, gethStartResult);
 
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
 

DistTestCore/GethStarter.cs

@@ -30,6 +30,17 @@ namespace DistTestCore
             return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
+        public GethStartResult BringOnlineValidatorFor(CodexSetup codexSetup, GethStartResult previousResult)
+        {
+            // allow marketplace and validator to be enabled on the same Codex node
+            if (previousResult.CompanionNode != null || (codexSetup.EnableValidator ?? false) == false) return previousResult;
+
+            var marketplaceNetwork = marketplaceNetworkCache.Get();
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
+
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
+        }
+
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
             if (marketplaceConfig.InitialTestTokens.Amount == 0) return;

Tests/Marketplace/Proofs.cs

@@ -0,0 +1,78 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class MarketplaceTests : DistTest
+    {
+        [Test]
+        public void HostThatMissesProofsIsPaidOutLessThanHostThatDoesNotMissProofs()
+        {
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+
+            var sellerWithFailures = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, new List<string>(){"market", "proving"})
+                .WithStorageQuota(11.GB())
+                .WithSimulateProofFailures(3)
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller with failures"));
+
+            var seller = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, new List<string>(){"market", "proving"})
+                .WithStorageQuota(11.GB())
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller"));
+
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace)
+                .WithBootstrapNode(seller)
+                .EnableMarketplace(buyerInitialBalance)
+                .WithName("buyer"));
+
+            var validator = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, new List<string>(){"validator"})
+                .WithBootstrapNode(seller)
+                .WithValidator()
+                .WithName("validator"));
+
+            seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
+            seller.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            sellerWithFailures.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var testFile = GenerateTestFile(10.MB());
+            var contentId = buyer.UploadFile(testFile);
+
+            buyer.Marketplace.RequestStorage(contentId,
+                pricePerBytePerSecond: 2.TestTokens(),
+                requiredCollateral: 10.TestTokens(),
+                minRequiredNumberOfNodes: 2,
+                proofProbability: 5,
+                duration: TimeSpan.FromMinutes(2));
+
+            // Time.Sleep(TimeSpan.FromMinutes(1));
+
+            // seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+
+            Time.Sleep(TimeSpan.FromMinutes(3));
+
+            var sellerBalance = seller.Marketplace.GetBalance();
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.LessThan(sellerBalance), "Seller that was slashed should have less balance than seller that was not slashed.");
+        }
+    }
+}