Debugging marketplace test

ContinuousTests/Tests/MarketplaceTest.cs

@@ -3,6 +3,7 @@ using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 
 namespace ContinuousTests.Tests
@@ -14,15 +15,14 @@ namespace ContinuousTests.Tests
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
+        public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
 
-        private const string MarketplaceTestNamespace = "codex-continuous-marketplace";
+        private readonly uint numberOfSlots = 3;
-
+        private readonly ByteSize fileSize = 10.MB();
-        private readonly ByteSize fileSize = 100.MB();
+        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
-        private readonly TestToken pricePerBytePerSecond = 1.TestTokens();
 
         private TestFile file = null!;
         private ContentId? cid;
-        private TestToken startingBalance = null!;
         private string purchaseId = string.Empty;
 
         [TestMoment(t: Zero)]
@@ -30,22 +30,28 @@ namespace ContinuousTests.Tests
         {
             var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
             decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
-            var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds;
+            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
+            Log.Log("expected total cost: " + expectedTotalCost);
 
             file = FileManager.GenerateTestFile(fileSize);
 
             var (workflowCreator, lifecycle) = CreateFacilities();
             var flow = workflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
                 var startupConfig = new StartupConfig();
                 var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
                 codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
                 codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
                 startupConfig.Add(codexStartConfig);
                 startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
                 var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
 
-            try
-            {
                 var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
                 var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
 
@@ -60,21 +66,24 @@ namespace ContinuousTests.Tests
                 cid = UploadFile(codexAccess.Node, file);
                 Assert.That(cid, Is.Not.Null);
 
-                startingBalance = marketAccess.GetBalance();
+                var balance = marketAccess.GetBalance();
+                Log.Log("Account: " + account.Account);
+                Log.Log("Balance: " + balance);
 
                 purchaseId = marketAccess.RequestStorage(
                     contentId: cid!,
-                    pricePerBytePerSecond: pricePerBytePerSecond,
+                    pricePerSlotPerSecond: pricePerSlotPerSecond,
                     requiredCollateral: 100.TestTokens(),
-                    minRequiredNumberOfNodes: 3,
+                    minRequiredNumberOfNodes: numberOfSlots,
                     proofProbability: 10,
                     duration: contractDuration);
 
+                Log.Log($"PurchaseId: '{purchaseId}'");
                 Assert.That(!string.IsNullOrEmpty(purchaseId));
             }
             finally
             {
-                flow.Stop(rc);
+                flow.DeleteTestResources();
             }
         }
 
@@ -83,13 +92,17 @@ namespace ContinuousTests.Tests
         {
             var (workflowCreator, lifecycle) = CreateFacilities();
             var flow = workflowCreator.CreateWorkflow();
-            var startupConfig = new StartupConfig();
-            var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
-            startupConfig.Add(codexStartConfig);
-            var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
           
             try
             {
+                var debugInfo = Nodes[0].GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
                 var container = rc.Containers[0];
                 var codexAccess = new CodexAccess(lifecycle, container);
 
@@ -99,32 +112,39 @@ namespace ContinuousTests.Tests
             }
             finally
             {
-                flow.Stop(rc);
+                flow.DeleteTestResources();
             }
         }
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
+            var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
             (
-                kubeConfigFile: Configuration.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 logPath: "null",
                 logDebug: false,
-                dataFilesPath: "notUsed",
+                dataFilesPath: Configuration.LogPath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: TestRunnerLocation.InternalToCluster
+                runnerLocation: TestRunnerLocation.ExternalToCluster
             );
 
-            var kubeConfig = new KubernetesWorkflow.Configuration(
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: MarketplaceTestNamespace,
-                kubeConfigFile: Configuration.KubeConfigFile,
+                kubeConfigFile: kubeConfig,
                 operationTimeout: TimeSet.K8sOperationTimeout(),
             retryDelay: TimeSet.WaitForK8sServiceDelay());
 
-            var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty);
+            var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
             var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
 
             return (workflowCreator, lifecycle);
         }
+
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
     }
 }

DistTestCore/Codex/CodexNode.cs

@@ -65,6 +65,11 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
+        }
+
         public string ConnectToPeer(string peerId, string peerMultiAddress)
         {
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
@@ -170,4 +175,10 @@ namespace DistTestCore.Codex
         public uint? nodes { get; set; }
         public uint? tolerance { get; set; }
     }
+
+    public class CodexStoragePurchase
+    {
+        public string state { get; set; } = string.Empty;
+        public string error { get; set; } = string.Empty;
+    }
 }

DistTestCore/DistTest.cs

@@ -250,7 +250,7 @@ namespace DistTestCore
         {
             OnEachCodexNode(lifecycle, node =>
             {
-                lifecycle.DownloadLog(node);
+                lifecycle.DownloadLog(node.CodexAccess.Container);
             });
         }
 

DistTestCore/Logs/DownloadedLog.cs

@@ -3,17 +3,17 @@ using NUnit.Framework;
 
 namespace DistTestCore.Logs
 {
-    public interface ICodexNodeLog
+    public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
     }
 
-    public class CodexNodeLog : ICodexNodeLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
-        private readonly OnlineCodexNode owner;
+        private readonly string owner;
 
-        public CodexNodeLog(LogFile logFile, OnlineCodexNode owner)
+        public DownloadedLog(LogFile logFile, string owner)
         {
             this.logFile = logFile;
             this.owner = owner;
@@ -31,7 +31,7 @@ namespace DistTestCore.Logs
                 line = streamReader.ReadLine();
             }
 
-            Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
+            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
     }
 }

DistTestCore/Logs/LogDownloadHandler.cs

@@ -5,21 +5,21 @@ namespace DistTestCore.Logs
 {
     public class LogDownloadHandler : LogHandler, ILogHandler
     {
-        private readonly OnlineCodexNode node;
+        private readonly RunningContainer container;
         private readonly LogFile log;
 
-        public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log)
+        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
         {
-            this.node = node;
+            this.container = container;
             this.log = log;
 
             log.Write($"{description} -->> {log.FullFilename}");
             log.WriteRaw(description);
         }
 
-        public CodexNodeLog CreateCodexNodeLog()
+        public DownloadedLog DownloadLog()
         {
-            return new CodexNodeLog(log, node);
+            return new DownloadedLog(log, container.Name);
         }
 
         protected override void ProcessLine(string line)

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToHexBigInt(duration.TotalSeconds),
                 proofProbability = ToHexBigInt(proofProbability),
-                reward = ToHexBigInt(pricePerBytePerSecond),
+                reward = ToHexBigInt(pricePerSlotPerSecond),
                 collateral = ToHexBigInt(requiredCollateral),
                 expiry = null,
                 nodes = minRequiredNumberOfNodes,
@@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
             };
 
             Log($"Requesting storage for: {contentId.Id}... (" +
-                $"pricePerBytePerSecond: {pricePerBytePerSecond}, " +
+                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +

DistTestCore/OnlineCodexNode.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
+        IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         ICodexSetup BringOffline();
@@ -107,9 +107,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public ICodexNodeLog DownloadLog()
+        public IDownloadedLog DownloadLog()
         {
-            return lifecycle.DownloadLog(this);
+            return lifecycle.DownloadLog(CodexAccess.Container);
         }
 
         public ICodexSetup BringOffline()

DistTestCore/TestLifecycle.cs

@@ -41,16 +41,16 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public ICodexNodeLog DownloadLog(OnlineCodexNode node)
+        public IDownloadedLog DownloadLog(RunningContainer container)
         {
             var subFile = Log.CreateSubfile();
-            var description = node.GetName();
+            var description = container.Name;
-            var handler = new LogDownloadHandler(node, description, subFile);
+            var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
+            CodexStarter.DownloadLog(container, handler);
 
-            return new CodexNodeLog(subFile, node);
+            return new DownloadedLog(subFile, description);
         }
 
         public string GetTestDuration()

KubernetesWorkflow/K8sController.cs

@@ -39,7 +39,7 @@ namespace KubernetesWorkflow
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
             var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
         }
 
         public void Stop(RunningPod pod)
@@ -436,9 +436,9 @@ namespace KubernetesWorkflow
 
         #region Service management
 
-        private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
+        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
         {
-            var result = new Dictionary<ContainerRecipe, Port[]>();
+            var result = new List<ContainerRecipePortMapEntry>();
 
             var ports = CreateServicePorts(containerRecipes);
 
@@ -468,7 +468,7 @@ namespace KubernetesWorkflow
             return (serviceSpec.Metadata.Name, result);
         }
 
-        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
         {
             // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@@ -485,7 +485,8 @@ namespace KubernetesWorkflow
                         // These service ports belongs to this recipe.
                         var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
                         var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
-                        result.Add(r, ports);
+
+                        result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
                     }
                 }
             }

KubernetesWorkflow/RunningPod.cs

@@ -2,35 +2,50 @@
 {
     public class RunningPod
     {
-        private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
-
-        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
             PodInfo = podInfo;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
-            this.servicePortMap = servicePortMap;
+            PortMapEntries = portMapEntries;
         }
 
         public K8sCluster Cluster { get; }
         public PodInfo PodInfo { get; }
+        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
-            if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
+            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
-            return servicePortMap[containerRecipe];
+            {
+                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
             }
+
+            return Array.Empty<Port>();
+        }
+    }
+
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
+        {
+            ContainerNumber = containerNumber;
+            Ports = ports;
+        }
+
+        public int ContainerNumber { get; }
+        public Port[] Ports { get; }
     }
 
     public class PodInfo
     {
-        public PodInfo(string podName, string podIp, string k8sNodeName)
+        public PodInfo(string name, string ip, string k8sNodeName)
         {
-            Name = podName;
+            Name = name;
-            Ip = podIp;
+            Ip = ip;
             K8SNodeName = k8sNodeName;
         }
 

Tests/BasicTests/ExampleTests.cs

@@ -64,7 +64,7 @@ namespace Tests.BasicTests
             
             var contentId = buyer.UploadFile(testFile);
             buyer.Marketplace.RequestStorage(contentId,
-                pricePerBytePerSecond: 2.TestTokens(),
+                pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,