Merge branch 'master' into feature/continuous-testing

CodexNetDeployer/Configuration.cs

@@ -8,16 +8,16 @@ namespace CodexNetDeployer
     public class Configuration
     {
         [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
-        public string CodexImage { get; set; } = string.Empty;
+        public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
 
         [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")]
-        public string GethImage { get; set; } = string.Empty;
+        public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
 
         [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")]
-        public string ContractsImage { get; set; } = string.Empty;
+        public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
 
-        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")]
+        [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
-        public string KubeConfigFile { get; set; } = string.Empty;
+        public string KubeConfigFile { get; set; } = "null";
 
         [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
         public string KubeNamespace { get; set; } = string.Empty;
@@ -32,18 +32,10 @@ namespace CodexNetDeployer
         public int? StorageQuota { get; set; }
 
         [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
-        public CodexLogLevel CodexLogLevel { get; set; }
+        public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
 
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
 
-        public class Defaults
-        {
-            public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
-            public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
-            public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
-            public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
-        }
-
         public List<string> Validate()
         {
             var errors = new List<string>();
@@ -74,7 +66,7 @@ namespace CodexNetDeployer
         {
             if (value == null || value.Value < 1)
             {
-                errors.Add($"{variable} is must be set and must be greater than 0.");
+                errors.Add($"{variable} must be set and must be greater than 0.");
             }
         }
 
@@ -82,7 +74,7 @@ namespace CodexNetDeployer
         {
             if (string.IsNullOrWhiteSpace(value))
             {
-                errors.Add($"{variable} is must be set.");
+                errors.Add($"{variable} must be set.");
             }
         }
     }

CodexNetDeployer/Deployer.cs

@@ -51,9 +51,11 @@ namespace CodexNetDeployer
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
+            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+
             var lifecycleConfig = new DistTestCore.Configuration
             (
-                kubeConfigFile: config.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 logPath: "null",
                 logDebug: false,
                 dataFilesPath: "notUsed",
@@ -61,18 +63,24 @@ namespace CodexNetDeployer
                 runnerLocation: config.RunnerLocation
             );
 
-            var kubeConfig = new KubernetesWorkflow.Configuration(
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: config.KubeNamespace,
-                kubeConfigFile: config.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 operationTimeout: timeset.K8sOperationTimeout(),
                 retryDelay: timeset.WaitForK8sServiceDelay());
 
-            var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty);
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
             var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
 
             return (workflowCreator, lifecycle);
         }
 
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(

CodexNetDeployer/Program.cs

@@ -17,7 +17,7 @@ public class Program
             return;
         }
 
-        var uniformArgs = new ArgsUniform<Configuration>(new Configuration.Defaults(), args);
+        var uniformArgs = new ArgsUniform<Configuration>(args);
         var config = uniformArgs.Parse(true);
         
         if (args.Any(a => a == "--external"))

ContinuousTests/Configuration.cs

@@ -18,8 +18,8 @@ namespace ContinuousTests
         [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
         public bool KeepPassedTestLogs { get; set; } = false;
 
-        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")]
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
-        public string KubeConfigFile { get; set; } = string.Empty;
+        public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
     }
@@ -30,7 +30,7 @@ namespace ContinuousTests
         {
             var uniformArgs = new ArgsUniform<Configuration>(args);
 
-            var result = uniformArgs.Parse();
+            var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
 

ContinuousTests/ContinuousTestRunner.cs

@@ -20,15 +20,18 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
+            overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
-            var testStarters = allTests.Select(t => new TestStarter(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
+            var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
 
-            foreach (var t in testStarters)
+            foreach (var t in testLoop)
             {
+                overviewLog.Log("Launching test-loop for " + t.Name);
                 t.Begin();
                 Thread.Sleep(TimeSpan.FromMinutes(5));
             }
 
+            overviewLog.Log("All test-loops launched.");
             while (true) Thread.Sleep((2 ^ 31) - 1);
         }
     }

ContinuousTests/ContinuousTests.csproj

@@ -14,6 +14,7 @@
   <ItemGroup>
     <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
   </ItemGroup>
 

ContinuousTests/SingleTestRun.cs

@@ -4,6 +4,7 @@ using Logging;
 using Utils;
 using KubernetesWorkflow;
 using NUnit.Framework.Internal;
+using System.Reflection;
 
 namespace ContinuousTests
 {
@@ -41,6 +42,22 @@ namespace ContinuousTests
                 try
                 {
                     RunTest();
+                    fileManager.DeleteAllTestFiles();
+                    Directory.Delete(dataFolder, true);
+                }
+                catch (Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
+        }
+
+        private void RunTest()
+        {
+            try
+            {
+                RunTestMoments();
 
                 if (!config.KeepPassedTestLogs) fixtureLog.Delete();
             }
@@ -49,12 +66,9 @@ namespace ContinuousTests
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
             }
-                fileManager.DeleteAllTestFiles();
-                Directory.Delete(dataFolder, true);
-            });
         }
 
-        private void RunTest()
+        private void RunTestMoments()
         {
             var earliestMoment = handle.GetEarliestMoment();
 
@@ -66,7 +80,7 @@ namespace ContinuousTests
                 if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
                 {
                     Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
-                    throw exceptions.Single();
+                    ThrowFailTest();
                 }
 
                 var nextMoment = handle.GetNextMoment(t);
@@ -80,9 +94,7 @@ namespace ContinuousTests
                 {
                     if (exceptions.Any())
                     {
-                        var ex = exceptions.First();
+                        ThrowFailTest();
-                        OverviewLog(" > Test failed: " + ex);
-                        throw ex;
                     }
                     OverviewLog(" > Test passed.");
                     return;
@@ -90,6 +102,28 @@ namespace ContinuousTests
             }
         }
 
+        private void ThrowFailTest()
+        {
+            var ex = UnpackException(exceptions.First());
+            Log(ex.ToString());
+            OverviewLog(" > Test failed: " + ex.Message);
+            throw ex;
+        }
+
+        private Exception UnpackException(Exception exception)
+        {
+            if (exception is AggregateException a)
+            {
+                return UnpackException(a.InnerExceptions.First());
+            }
+            if (exception is TargetInvocationException t)
+            {
+                return UnpackException(t.InnerException!);
+            }
+
+            return exception;
+        }
+
         private void RunMoment(int t)
         {
             using (var context = new TestExecutionContext.IsolatedContext())
@@ -100,7 +134,6 @@ namespace ContinuousTests
                 }
                 catch (Exception ex)
                 {
-                    Log($" > TestMoment yielded exception: " + ex);
                     exceptions.Add(ex);
                 }
             }

ContinuousTests/TestLoop.cs

@@ -2,30 +2,42 @@
 
 namespace ContinuousTests
 {
-    public class TestStarter
+    public class TestLoop
     {
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
 
-        public TestStarter(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
+        public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
         {
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
+
+            Name = testType.Name;
         }
 
+        public string Name { get; }
+
         public void Begin()
         {
             Task.Run(() =>
+            {
+                try
                 {
                     while (true)
                     {
                         StartTest();
                         Thread.Sleep(runsEvery);
                     }
+                }
+                catch(Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
+                    Environment.Exit(-1);
+                }
             });
         }
 

ContinuousTests/Tests/MarketplaceTest.cs

@@ -3,6 +3,7 @@ using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 
 namespace ContinuousTests.Tests
@@ -11,18 +12,17 @@ namespace ContinuousTests.Tests
     {
         public override int RequiredNumberOfNodes => 1;
         public override TimeSpan RunTestEvery => TimeSpan.FromDays(4);
-        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
+        public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
 
-        private const string MarketplaceTestNamespace = "codex-continuous-marketplace";
+        private readonly uint numberOfSlots = 3;
-
+        private readonly ByteSize fileSize = 10.MB();
-        private readonly ByteSize fileSize = 100.MB();
+        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
-        private readonly TestToken pricePerBytePerSecond = 1.TestTokens();
 
         private TestFile file = null!;
         private ContentId? cid;
-        private TestToken startingBalance = null!;
         private string purchaseId = string.Empty;
 
         [TestMoment(t: Zero)]
@@ -30,22 +30,28 @@ namespace ContinuousTests.Tests
         {
             var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
             decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
-            var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds;
+            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
+            Log.Log("expected total cost: " + expectedTotalCost);
 
             file = FileManager.GenerateTestFile(fileSize);
 
             var (workflowCreator, lifecycle) = CreateFacilities();
             var flow = workflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
                 var startupConfig = new StartupConfig();
                 var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
                 codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
                 codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
                 startupConfig.Add(codexStartConfig);
                 startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
                 var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
 
-            try
-            {
                 var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
                 var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
 
@@ -60,21 +66,24 @@ namespace ContinuousTests.Tests
                 cid = UploadFile(codexAccess.Node, file);
                 Assert.That(cid, Is.Not.Null);
 
-                startingBalance = marketAccess.GetBalance();
+                var balance = marketAccess.GetBalance();
+                Log.Log("Account: " + account.Account);
+                Log.Log("Balance: " + balance);
 
                 purchaseId = marketAccess.RequestStorage(
                     contentId: cid!,
-                    pricePerBytePerSecond: pricePerBytePerSecond,
+                    pricePerSlotPerSecond: pricePerSlotPerSecond,
                     requiredCollateral: 100.TestTokens(),
-                    minRequiredNumberOfNodes: 3,
+                    minRequiredNumberOfNodes: numberOfSlots,
                     proofProbability: 10,
                     duration: contractDuration);
 
+                Log.Log($"PurchaseId: '{purchaseId}'");
                 Assert.That(!string.IsNullOrEmpty(purchaseId));
             }
             finally
             {
-                flow.Stop(rc);
+                flow.DeleteTestResources();
             }
         }
 
@@ -83,13 +92,17 @@ namespace ContinuousTests.Tests
         {
             var (workflowCreator, lifecycle) = CreateFacilities();
             var flow = workflowCreator.CreateWorkflow();
-            var startupConfig = new StartupConfig();
-            var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
-            startupConfig.Add(codexStartConfig);
-            var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
           
             try
             {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
                 var container = rc.Containers[0];
                 var codexAccess = new CodexAccess(lifecycle, container);
 
@@ -99,32 +112,39 @@ namespace ContinuousTests.Tests
             }
             finally
             {
-                flow.Stop(rc);
+                flow.DeleteTestResources();
             }
         }
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
+            var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
             (
-                kubeConfigFile: Configuration.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 logPath: "null",
                 logDebug: false,
-                dataFilesPath: "notUsed",
+                dataFilesPath: Configuration.LogPath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: TestRunnerLocation.InternalToCluster
+                runnerLocation: TestRunnerLocation.ExternalToCluster
             );
 
-            var kubeConfig = new KubernetesWorkflow.Configuration(
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: MarketplaceTestNamespace,
-                kubeConfigFile: Configuration.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 operationTimeout: TimeSet.K8sOperationTimeout(),
             retryDelay: TimeSet.WaitForK8sServiceDelay());
 
-            var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty);
+            var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
             var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
 
             return (workflowCreator, lifecycle);
         }
+
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
     }
 }

DistTestCore/Codex/CodexNode.cs

@@ -65,6 +65,11 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
+        }
+
         public string ConnectToPeer(string peerId, string peerMultiAddress)
         {
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
@@ -170,4 +175,10 @@ namespace DistTestCore.Codex
         public uint? nodes { get; set; }
         public uint? tolerance { get; set; }
     }
+
+    public class CodexStoragePurchase
+    {
+        public string state { get; set; } = string.Empty;
+        public string error { get; set; } = string.Empty;
+    }
 }

DistTestCore/DistTest.cs

@@ -250,7 +250,7 @@ namespace DistTestCore
         {
             OnEachCodexNode(lifecycle, node =>
             {
-                lifecycle.DownloadLog(node);
+                lifecycle.DownloadLog(node.CodexAccess.Container);
             });
         }
 

DistTestCore/Logs/DownloadedLog.cs

@@ -3,17 +3,17 @@ using NUnit.Framework;
 
 namespace DistTestCore.Logs
 {
-    public interface ICodexNodeLog
+    public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
     }
 
-    public class CodexNodeLog : ICodexNodeLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
-        private readonly OnlineCodexNode owner;
+        private readonly string owner;
 
-        public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
+        public DownloadedLog(LogFile logFile, string owner)
         {
             this.logFile = logFile;
             this.owner = owner;
@@ -31,7 +31,7 @@ namespace DistTestCore.Logs
                 line = streamReader.ReadLine();
             }
 
-            Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
+            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
     }
 }

DistTestCore/Logs/LogDownloadHandler.cs

@@ -5,21 +5,21 @@ namespace DistTestCore.Logs
 {
     public class LogDownloadHandler : LogHandler, ILogHandler
     {
-        private readonly OnlineCodexNode node;
+        private readonly RunningContainer container;
         private readonly LogFile log;
 
-        public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
+        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
         {
-            this.node = node;
+            this.container = container;
             this.log = log;
 
             log.Write($"{description} -->> {log.FullFilename}");
             log.WriteRaw(description);
         }
 
-        public CodexNodeLog CreateCodexNodeLog()
+        public DownloadedLog DownloadLog()
         {
-            return new CodexNodeLog(log, node);
+            return new DownloadedLog(log, container.Name);
         }
 
         protected override void ProcessLine(string line)

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToHexBigInt(duration.TotalSeconds),
                 proofProbability = ToHexBigInt(proofProbability),
-                reward = ToHexBigInt(pricePerBytePerSecond),
+                reward = ToHexBigInt(pricePerSlotPerSecond),
                 collateral = ToHexBigInt(requiredCollateral),
                 expiry = null,
                 nodes = minRequiredNumberOfNodes,
@@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
             };
 
             Log($"Requesting storage for: {contentId.Id}... (" +
-                $"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
+                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +

DistTestCore/OnlineCodexNode.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
+        IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         ICodexSetup BringOffline();
@@ -107,9 +107,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public ICodexNodeLog DownloadLog()
+        public IDownloadedLog DownloadLog()
         {
-            return lifecycle.DownloadLog(this);
+            return lifecycle.DownloadLog(CodexAccess.Container);
         }
 
         public ICodexSetup BringOffline()

DistTestCore/TestLifecycle.cs

@@ -41,16 +41,16 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public ICodexNodeLog DownloadLog(OnlineCodexNode node)
+        public IDownloadedLog DownloadLog(RunningContainer container)
         {
             var subFile = Log.CreateSubfile();
-            var description = node.GetName();
+            var description = container.Name;
-            var handler = new LogDownloadHandler(node, description, subFile);
+            var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
+            CodexStarter.DownloadLog(container, handler);
 
-            return new CodexNodeLog(subFile, node);
+            return new DownloadedLog(subFile, description);
         }
 
         public string GetTestDuration()

KubernetesWorkflow/K8sController.cs

@@ -39,7 +39,7 @@ namespace KubernetesWorkflow
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
             var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
         }
 
         public void Stop(RunningPod pod)
@@ -436,9 +436,9 @@ namespace KubernetesWorkflow
 
         #region Service management
 
-        private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
+        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
         {
-            var result = new Dictionary<ContainerRecipe, Port[]>();
+            var result = new List<ContainerRecipePortMapEntry>();
 
             var ports = CreateServicePorts(containerRecipes);
 
@@ -468,7 +468,7 @@ namespace KubernetesWorkflow
             return (serviceSpec.Metadata.Name, result);
         }
 
-        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
         {
             // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@@ -485,7 +485,8 @@ namespace KubernetesWorkflow
                         // These service ports belongs to this recipe.
                         var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
-                        result.Add(r, ports);
+
+                        result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
                     }
                 }
             }

KubernetesWorkflow/RunningPod.cs

@@ -2,35 +2,50 @@
 {
     public class RunningPod
     {
-        private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
-
-        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
             PodInfo = podInfo;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
-            this.servicePortMap = servicePortMap;
+            PortMapEntries = portMapEntries;
         }
 
         public K8sCluster Cluster { get; }
         public PodInfo PodInfo { get; }
+        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
+            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
-            return servicePortMap[containerRecipe];
+            {
+                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
             }
+
+            return Array.Empty<Port>();
+        }
+    }
+
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
+        {
+            ContainerNumber = containerNumber;
+            Ports = ports;
+        }
+
+        public int ContainerNumber { get; }
+        public Port[] Ports { get; }
     }
 
     public class PodInfo
     {
-        public PodInfo(string podName, string podIp, string k8sNodeName)
+        public PodInfo(string name, string ip, string k8sNodeName)
         {
-            Name = podName;
+            Name = name;
-            Ip = podIp;
+            Ip = ip;
             K8SNodeName = k8sNodeName;
         }
 

Tests/BasicTests/ExampleTests.cs

@@ -64,7 +64,7 @@ namespace Tests.BasicTests
             
             var contentId = buyer.UploadFile(testFile);
             buyer.Marketplace.RequestStorage(contentId,
-                pricePerBytePerSecond: 2.TestTokens(),
+                pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,