diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 3702732..c99bcd1 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -3,6 +3,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; using Logging; +using Newtonsoft.Json; using NUnit.Framework; namespace ContinuousTests.Tests @@ -14,15 +15,14 @@ namespace ContinuousTests.Tests public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. + public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too - private const string MarketplaceTestNamespace = "codex-continuous-marketplace"; - - private readonly ByteSize fileSize = 100.MB(); - private readonly TestToken pricePerBytePerSecond = 1.TestTokens(); + private readonly uint numberOfSlots = 3; + private readonly ByteSize fileSize = 10.MB(); + private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); private TestFile file = null!; private ContentId? cid; - private TestToken startingBalance = null!; private string purchaseId = string.Empty; [TestMoment(t: Zero)] @@ -30,22 +30,28 @@ namespace ContinuousTests.Tests { var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1); decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); - var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds; + var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1); + Log.Log("expected total cost: " + expectedTotalCost); file = FileManager.GenerateTestFile(fileSize); var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); - codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); - codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; - startupConfig.Add(codexStartConfig); - startupConfig.Add(Configuration.CodexDeployment.GethStartResult); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); try { + var debugInfo = Nodes[0].GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); + + var startupConfig = new StartupConfig(); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); + codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false); + codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex; + codexStartConfig.BootstrapSpr = debugInfo.spr; + startupConfig.Add(codexStartConfig); + startupConfig.Add(Configuration.CodexDeployment.GethStartResult); + var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); + var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex]; var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress; @@ -60,21 +66,24 @@ namespace ContinuousTests.Tests cid = UploadFile(codexAccess.Node, file); Assert.That(cid, Is.Not.Null); - startingBalance = marketAccess.GetBalance(); + var balance = marketAccess.GetBalance(); + Log.Log("Account: " + account.Account); + Log.Log("Balance: " + balance); purchaseId = marketAccess.RequestStorage( contentId: cid!, - pricePerBytePerSecond: pricePerBytePerSecond, + pricePerSlotPerSecond: pricePerSlotPerSecond, requiredCollateral: 100.TestTokens(), - minRequiredNumberOfNodes: 3, + minRequiredNumberOfNodes: numberOfSlots, proofProbability: 10, duration: contractDuration); + Log.Log($"PurchaseId: '{purchaseId}'"); Assert.That(!string.IsNullOrEmpty(purchaseId)); } finally { - flow.Stop(rc); + flow.DeleteTestResources(); } } @@ -83,13 +92,17 @@ namespace ContinuousTests.Tests { var (workflowCreator, lifecycle) = CreateFacilities(); var flow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); - startupConfig.Add(codexStartConfig); - var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); - + try { + var debugInfo = Nodes[0].GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(debugInfo.spr)); + + var startupConfig = new StartupConfig(); + var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug); + codexStartConfig.BootstrapSpr = debugInfo.spr; + startupConfig.Add(codexStartConfig); + var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig); var container = rc.Containers[0]; var codexAccess = new CodexAccess(lifecycle, container); @@ -99,32 +112,39 @@ namespace ContinuousTests.Tests } finally { - flow.Stop(rc); + flow.DeleteTestResources(); } } private (WorkflowCreator, TestLifecycle) CreateFacilities() { + var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: Configuration.KubeConfigFile, + kubeConfigFile: kubeConfig, logPath: "null", - logDebug: false, - dataFilesPath: "notUsed", + logDebug: false, + dataFilesPath: Configuration.LogPath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.InternalToCluster + runnerLocation: TestRunnerLocation.ExternalToCluster ); - var kubeConfig = new KubernetesWorkflow.Configuration( + var kubeFlowConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: MarketplaceTestNamespace, - kubeConfigFile: Configuration.KubeConfigFile, + kubeConfigFile: kubeConfig, operationTimeout: TimeSet.K8sOperationTimeout(), retryDelay: TimeSet.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty); var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator); return (workflowCreator, lifecycle); } + + private string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } } } diff --git a/DistTestCore/Codex/CodexNode.cs b/DistTestCore/Codex/CodexNode.cs index 510c2fb..bb2220e 100644 --- a/DistTestCore/Codex/CodexNode.cs +++ b/DistTestCore/Codex/CodexNode.cs @@ -65,6 +65,11 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + { + return Http().HttpGetJson($"storage/purchases/{purchaseId}"); + } + public string ConnectToPeer(string peerId, string peerMultiAddress) { return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); @@ -170,4 +175,10 @@ namespace DistTestCore.Codex public uint? nodes { get; set; } public uint? tolerance { get; set; } } + + public class CodexStoragePurchase + { + public string state { get; set; } = string.Empty; + public string error { get; set; } = string.Empty; + } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a93c180..e2a5c77 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -250,7 +250,7 @@ namespace DistTestCore { OnEachCodexNode(lifecycle, node => { - lifecycle.DownloadLog(node); + lifecycle.DownloadLog(node.CodexAccess.Container); }); } diff --git a/DistTestCore/Logs/CodexNodeLog.cs b/DistTestCore/Logs/DownloadedLog.cs similarity index 68% rename from DistTestCore/Logs/CodexNodeLog.cs rename to DistTestCore/Logs/DownloadedLog.cs index 6dd658f..9d22c81 100644 --- a/DistTestCore/Logs/CodexNodeLog.cs +++ b/DistTestCore/Logs/DownloadedLog.cs @@ -3,17 +3,17 @@ using NUnit.Framework; namespace DistTestCore.Logs { - public interface ICodexNodeLog + public interface IDownloadedLog { void AssertLogContains(string expectedString); } - public class CodexNodeLog : ICodexNodeLog + public class DownloadedLog : IDownloadedLog { private readonly LogFile logFile; - private readonly OnlineCodexNode owner; + private readonly string owner; - public CodexNodeLog(LogFile logFile, OnlineCodexNode owner) + public DownloadedLog(LogFile logFile, string owner) { this.logFile = logFile; this.owner = owner; @@ -31,7 +31,7 @@ namespace DistTestCore.Logs line = streamReader.ReadLine(); } - Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); + Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } } } diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index 2c7dc9f..483e46b 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -5,21 +5,21 @@ namespace DistTestCore.Logs { public class LogDownloadHandler : LogHandler, ILogHandler { - private readonly OnlineCodexNode node; + private readonly RunningContainer container; private readonly LogFile log; - public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log) + public LogDownloadHandler(RunningContainer container, string description, LogFile log) { - this.node = node; + this.container = container; this.log = log; log.Write($"{description} -->> {log.FullFilename}"); log.WriteRaw(description); } - public CodexNodeLog CreateCodexNodeLog() + public DownloadedLog DownloadLog() { - return new CodexNodeLog(log, node); + return new DownloadedLog(log, container.Name); } protected override void ProcessLine(string line) diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 29730a6..f64e271 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace public interface IMarketplaceAccess { string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); - string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); + string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); void AssertThatBalance(IResolveConstraint constraint, string message = ""); TestToken GetBalance(); } @@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace this.codexAccess = codexAccess; } - public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) + public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) { var request = new CodexSalesRequestStorageRequest { duration = ToHexBigInt(duration.TotalSeconds), proofProbability = ToHexBigInt(proofProbability), - reward = ToHexBigInt(pricePerBytePerSecond), + reward = ToHexBigInt(pricePerSlotPerSecond), collateral = ToHexBigInt(requiredCollateral), expiry = null, nodes = minRequiredNumberOfNodes, @@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace }; Log($"Requesting storage for: {contentId.Id}... (" + - $"pricePerBytePerSecond: {pricePerBytePerSecond}, " + + $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " + $"requiredCollateral: {requiredCollateral}, " + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " + $"proofProbability: {proofProbability}, " + diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index adc7dfc..d1e5301 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -16,7 +16,7 @@ namespace DistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); - ICodexNodeLog DownloadLog(); + IDownloadedLog DownloadLog(); IMetricsAccess Metrics { get; } IMarketplaceAccess Marketplace { get; } ICodexSetup BringOffline(); @@ -107,9 +107,9 @@ namespace DistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - public ICodexNodeLog DownloadLog() + public IDownloadedLog DownloadLog() { - return lifecycle.DownloadLog(this); + return lifecycle.DownloadLog(CodexAccess.Container); } public ICodexSetup BringOffline() diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 667b96c..6ea8441 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -41,16 +41,16 @@ namespace DistTestCore FileManager.DeleteAllTestFiles(); } - public ICodexNodeLog DownloadLog(OnlineCodexNode node) + public IDownloadedLog DownloadLog(RunningContainer container) { var subFile = Log.CreateSubfile(); - var description = node.GetName(); - var handler = new LogDownloadHandler(node, description, subFile); + var description = container.Name; + var handler = new LogDownloadHandler(container, description, subFile); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); - CodexStarter.DownloadLog(node.CodexAccess.Container, handler); + CodexStarter.DownloadLog(container, handler); - return new CodexNodeLog(subFile, node); + return new DownloadedLog(subFile, description); } public string GetTestDuration() diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 6174291..d679c8c 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -39,7 +39,7 @@ namespace KubernetesWorkflow var (serviceName, servicePortsMap) = CreateService(containerRecipes); var podInfo = FetchNewPod(); - return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap); + return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray()); } public void Stop(RunningPod pod) @@ -436,9 +436,9 @@ namespace KubernetesWorkflow #region Service management - private (string, Dictionary) CreateService(ContainerRecipe[] containerRecipes) + private (string, List) CreateService(ContainerRecipe[] containerRecipes) { - var result = new Dictionary(); + var result = new List(); var ports = CreateServicePorts(containerRecipes); @@ -468,7 +468,7 @@ namespace KubernetesWorkflow return (serviceSpec.Metadata.Name, result); } - private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary result) + private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List result) { // For each container-recipe, we need to figure out which service-ports it was assigned by K8s. var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace)); @@ -485,7 +485,8 @@ namespace KubernetesWorkflow // These service ports belongs to this recipe. var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName)); var ports = optionals.Where(p => p != null).Select(p => p!).ToArray(); - result.Add(r, ports); + + result.Add(new ContainerRecipePortMapEntry(r.Number, ports)); } } } diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 1618410..ca8c9c1 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -2,35 +2,50 @@ { public class RunningPod { - private readonly Dictionary servicePortMap; - - public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, ContainerRecipePortMapEntry[] portMapEntries) { Cluster = cluster; PodInfo = podInfo; DeploymentName = deploymentName; ServiceName = serviceName; - this.servicePortMap = servicePortMap; + PortMapEntries = portMapEntries; } public K8sCluster Cluster { get; } public PodInfo PodInfo { get; } + public ContainerRecipePortMapEntry[] PortMapEntries { get; } internal string DeploymentName { get; } internal string ServiceName { get; } public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { - if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty(); - return servicePortMap[containerRecipe]; + if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number)) + { + return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports; + } + + return Array.Empty(); } } + public class ContainerRecipePortMapEntry + { + public ContainerRecipePortMapEntry(int containerNumber, Port[] ports) + { + ContainerNumber = containerNumber; + Ports = ports; + } + + public int ContainerNumber { get; } + public Port[] Ports { get; } + } + public class PodInfo { - public PodInfo(string podName, string podIp, string k8sNodeName) + public PodInfo(string name, string ip, string k8sNodeName) { - Name = podName; - Ip = podIp; + Name = name; + Ip = ip; K8SNodeName = k8sNodeName; } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 976ccc3..3007ae4 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -64,7 +64,7 @@ namespace Tests.BasicTests var contentId = buyer.UploadFile(testFile); buyer.Marketplace.RequestStorage(contentId, - pricePerBytePerSecond: 2.TestTokens(), + pricePerSlotPerSecond: 2.TestTokens(), requiredCollateral: 10.TestTokens(), minRequiredNumberOfNodes: 1, proofProbability: 5,