Debugging marketplace test

This commit is contained in:
benbierens 2023-06-27 15:28:00 +02:00
parent 86d954e103
commit 63068aae1d
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
11 changed files with 115 additions and 68 deletions

View File

@ -3,6 +3,7 @@ using DistTestCore.Codex;
using DistTestCore.Marketplace; using DistTestCore.Marketplace;
using KubernetesWorkflow; using KubernetesWorkflow;
using Logging; using Logging;
using Newtonsoft.Json;
using NUnit.Framework; using NUnit.Framework;
namespace ContinuousTests.Tests namespace ContinuousTests.Tests
@ -14,15 +15,14 @@ namespace ContinuousTests.Tests
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests. public const int EthereumAccountIndex = 200; // TODO: Check against all other account indices of all other tests.
public const string MarketplaceTestNamespace = "codex-continuous-marketplace"; // prevent clashes too
private const string MarketplaceTestNamespace = "codex-continuous-marketplace"; private readonly uint numberOfSlots = 3;
private readonly ByteSize fileSize = 10.MB();
private readonly ByteSize fileSize = 100.MB(); private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
private readonly TestToken pricePerBytePerSecond = 1.TestTokens();
private TestFile file = null!; private TestFile file = null!;
private ContentId? cid; private ContentId? cid;
private TestToken startingBalance = null!;
private string purchaseId = string.Empty; private string purchaseId = string.Empty;
[TestMoment(t: Zero)] [TestMoment(t: Zero)]
@ -30,22 +30,28 @@ namespace ContinuousTests.Tests
{ {
var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1); var contractDuration = TimeSpan.FromDays(3) + TimeSpan.FromHours(1);
decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
var expectedTotalCost = pricePerBytePerSecond.Amount * totalDurationSeconds; var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1);
Log.Log("expected total cost: " + expectedTotalCost);
file = FileManager.GenerateTestFile(fileSize); file = FileManager.GenerateTestFile(fileSize);
var (workflowCreator, lifecycle) = CreateFacilities(); var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow(); var flow = workflowCreator.CreateWorkflow();
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
startupConfig.Add(codexStartConfig);
startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
try try
{ {
var debugInfo = Nodes[0].GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
codexStartConfig.MarketplaceConfig.AccountIndexOverride = EthereumAccountIndex;
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
startupConfig.Add(Configuration.CodexDeployment.GethStartResult);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex]; var account = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[EthereumAccountIndex];
var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress; var tokenAddress = Configuration.CodexDeployment.GethStartResult.MarketplaceNetwork.Marketplace.TokenAddress;
@ -60,21 +66,24 @@ namespace ContinuousTests.Tests
cid = UploadFile(codexAccess.Node, file); cid = UploadFile(codexAccess.Node, file);
Assert.That(cid, Is.Not.Null); Assert.That(cid, Is.Not.Null);
startingBalance = marketAccess.GetBalance(); var balance = marketAccess.GetBalance();
Log.Log("Account: " + account.Account);
Log.Log("Balance: " + balance);
purchaseId = marketAccess.RequestStorage( purchaseId = marketAccess.RequestStorage(
contentId: cid!, contentId: cid!,
pricePerBytePerSecond: pricePerBytePerSecond, pricePerSlotPerSecond: pricePerSlotPerSecond,
requiredCollateral: 100.TestTokens(), requiredCollateral: 100.TestTokens(),
minRequiredNumberOfNodes: 3, minRequiredNumberOfNodes: numberOfSlots,
proofProbability: 10, proofProbability: 10,
duration: contractDuration); duration: contractDuration);
Log.Log($"PurchaseId: '{purchaseId}'");
Assert.That(!string.IsNullOrEmpty(purchaseId)); Assert.That(!string.IsNullOrEmpty(purchaseId));
} }
finally finally
{ {
flow.Stop(rc); flow.DeleteTestResources();
} }
} }
@ -83,13 +92,17 @@ namespace ContinuousTests.Tests
{ {
var (workflowCreator, lifecycle) = CreateFacilities(); var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow(); var flow = workflowCreator.CreateWorkflow();
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
startupConfig.Add(codexStartConfig);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
try try
{ {
var debugInfo = Nodes[0].GetDebugInfo();
Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
var startupConfig = new StartupConfig();
var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Debug);
codexStartConfig.BootstrapSpr = debugInfo.spr;
startupConfig.Add(codexStartConfig);
var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
var container = rc.Containers[0]; var container = rc.Containers[0];
var codexAccess = new CodexAccess(lifecycle, container); var codexAccess = new CodexAccess(lifecycle, container);
@ -99,32 +112,39 @@ namespace ContinuousTests.Tests
} }
finally finally
{ {
flow.Stop(rc); flow.DeleteTestResources();
} }
} }
private (WorkflowCreator, TestLifecycle) CreateFacilities() private (WorkflowCreator, TestLifecycle) CreateFacilities()
{ {
var kubeConfig = GetKubeConfig(Configuration.KubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration var lifecycleConfig = new DistTestCore.Configuration
( (
kubeConfigFile: Configuration.KubeConfigFile, kubeConfigFile: kubeConfig,
logPath: "null", logPath: "null",
logDebug: false, logDebug: false,
dataFilesPath: "notUsed", dataFilesPath: Configuration.LogPath,
codexLogLevel: CodexLogLevel.Debug, codexLogLevel: CodexLogLevel.Debug,
runnerLocation: TestRunnerLocation.InternalToCluster runnerLocation: TestRunnerLocation.ExternalToCluster
); );
var kubeConfig = new KubernetesWorkflow.Configuration( var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: MarketplaceTestNamespace, k8sNamespacePrefix: MarketplaceTestNamespace,
kubeConfigFile: Configuration.KubeConfigFile, kubeConfigFile: kubeConfig,
operationTimeout: TimeSet.K8sOperationTimeout(), operationTimeout: TimeSet.K8sOperationTimeout(),
retryDelay: TimeSet.WaitForK8sServiceDelay()); retryDelay: TimeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(Log, kubeConfig, testNamespacePostfix: string.Empty); var workflowCreator = new WorkflowCreator(Log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator); var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, TimeSet, workflowCreator);
return (workflowCreator, lifecycle); return (workflowCreator, lifecycle);
} }
private string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
} }
} }

View File

@ -65,6 +65,11 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request); return Http().HttpPostJson($"storage/request/{contentId}", request);
} }
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
{
return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
}
public string ConnectToPeer(string peerId, string peerMultiAddress) public string ConnectToPeer(string peerId, string peerMultiAddress)
{ {
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
@ -170,4 +175,10 @@ namespace DistTestCore.Codex
public uint? nodes { get; set; } public uint? nodes { get; set; }
public uint? tolerance { get; set; } public uint? tolerance { get; set; }
} }
public class CodexStoragePurchase
{
public string state { get; set; } = string.Empty;
public string error { get; set; } = string.Empty;
}
} }

View File

@ -250,7 +250,7 @@ namespace DistTestCore
{ {
OnEachCodexNode(lifecycle, node => OnEachCodexNode(lifecycle, node =>
{ {
lifecycle.DownloadLog(node); lifecycle.DownloadLog(node.CodexAccess.Container);
}); });
} }

View File

@ -3,17 +3,17 @@ using NUnit.Framework;
namespace DistTestCore.Logs namespace DistTestCore.Logs
{ {
public interface ICodexNodeLog public interface IDownloadedLog
{ {
void AssertLogContains(string expectedString); void AssertLogContains(string expectedString);
} }
public class CodexNodeLog : ICodexNodeLog public class DownloadedLog : IDownloadedLog
{ {
private readonly LogFile logFile; private readonly LogFile logFile;
private readonly OnlineCodexNode owner; private readonly string owner;
public CodexNodeLog(LogFile logFile, OnlineCodexNode owner) public DownloadedLog(LogFile logFile, string owner)
{ {
this.logFile = logFile; this.logFile = logFile;
this.owner = owner; this.owner = owner;
@ -31,7 +31,7 @@ namespace DistTestCore.Logs
line = streamReader.ReadLine(); line = streamReader.ReadLine();
} }
Assert.Fail($"{owner.GetName()} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
} }
} }
} }

View File

@ -5,21 +5,21 @@ namespace DistTestCore.Logs
{ {
public class LogDownloadHandler : LogHandler, ILogHandler public class LogDownloadHandler : LogHandler, ILogHandler
{ {
private readonly OnlineCodexNode node; private readonly RunningContainer container;
private readonly LogFile log; private readonly LogFile log;
public LogDownloadHandler(OnlineCodexNode node, string description, LogFile log) public LogDownloadHandler(RunningContainer container, string description, LogFile log)
{ {
this.node = node; this.container = container;
this.log = log; this.log = log;
log.Write($"{description} -->> {log.FullFilename}"); log.Write($"{description} -->> {log.FullFilename}");
log.WriteRaw(description); log.WriteRaw(description);
} }
public CodexNodeLog CreateCodexNodeLog() public DownloadedLog DownloadLog()
{ {
return new CodexNodeLog(log, node); return new DownloadedLog(log, container.Name);
} }
protected override void ProcessLine(string line) protected override void ProcessLine(string line)

View File

@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
public interface IMarketplaceAccess public interface IMarketplaceAccess
{ {
string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration); string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration); string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
void AssertThatBalance(IResolveConstraint constraint, string message = ""); void AssertThatBalance(IResolveConstraint constraint, string message = "");
TestToken GetBalance(); TestToken GetBalance();
} }
@ -30,13 +30,13 @@ namespace DistTestCore.Marketplace
this.codexAccess = codexAccess; this.codexAccess = codexAccess;
} }
public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration) public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
{ {
var request = new CodexSalesRequestStorageRequest var request = new CodexSalesRequestStorageRequest
{ {
duration = ToHexBigInt(duration.TotalSeconds), duration = ToHexBigInt(duration.TotalSeconds),
proofProbability = ToHexBigInt(proofProbability), proofProbability = ToHexBigInt(proofProbability),
reward = ToHexBigInt(pricePerBytePerSecond), reward = ToHexBigInt(pricePerSlotPerSecond),
collateral = ToHexBigInt(requiredCollateral), collateral = ToHexBigInt(requiredCollateral),
expiry = null, expiry = null,
nodes = minRequiredNumberOfNodes, nodes = minRequiredNumberOfNodes,
@ -44,7 +44,7 @@ namespace DistTestCore.Marketplace
}; };
Log($"Requesting storage for: {contentId.Id}... (" + Log($"Requesting storage for: {contentId.Id}... (" +
$"pricePerBytePerSecond: {pricePerBytePerSecond}, " + $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
$"requiredCollateral: {requiredCollateral}, " + $"requiredCollateral: {requiredCollateral}, " +
$"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " + $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
$"proofProbability: {proofProbability}, " + $"proofProbability: {proofProbability}, " +

View File

@ -16,7 +16,7 @@ namespace DistTestCore
ContentId UploadFile(TestFile file); ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node); void ConnectToPeer(IOnlineCodexNode node);
ICodexNodeLog DownloadLog(); IDownloadedLog DownloadLog();
IMetricsAccess Metrics { get; } IMetricsAccess Metrics { get; }
IMarketplaceAccess Marketplace { get; } IMarketplaceAccess Marketplace { get; }
ICodexSetup BringOffline(); ICodexSetup BringOffline();
@ -107,9 +107,9 @@ namespace DistTestCore
Log($"Successfully connected to peer {peer.GetName()}."); Log($"Successfully connected to peer {peer.GetName()}.");
} }
public ICodexNodeLog DownloadLog() public IDownloadedLog DownloadLog()
{ {
return lifecycle.DownloadLog(this); return lifecycle.DownloadLog(CodexAccess.Container);
} }
public ICodexSetup BringOffline() public ICodexSetup BringOffline()

View File

@ -41,16 +41,16 @@ namespace DistTestCore
FileManager.DeleteAllTestFiles(); FileManager.DeleteAllTestFiles();
} }
public ICodexNodeLog DownloadLog(OnlineCodexNode node) public IDownloadedLog DownloadLog(RunningContainer container)
{ {
var subFile = Log.CreateSubfile(); var subFile = Log.CreateSubfile();
var description = node.GetName(); var description = container.Name;
var handler = new LogDownloadHandler(node, description, subFile); var handler = new LogDownloadHandler(container, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
CodexStarter.DownloadLog(node.CodexAccess.Container, handler); CodexStarter.DownloadLog(container, handler);
return new CodexNodeLog(subFile, node); return new DownloadedLog(subFile, description);
} }
public string GetTestDuration() public string GetTestDuration()

View File

@ -39,7 +39,7 @@ namespace KubernetesWorkflow
var (serviceName, servicePortsMap) = CreateService(containerRecipes); var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var podInfo = FetchNewPod(); var podInfo = FetchNewPod();
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap); return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
} }
public void Stop(RunningPod pod) public void Stop(RunningPod pod)
@ -436,9 +436,9 @@ namespace KubernetesWorkflow
#region Service management #region Service management
private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes) private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
{ {
var result = new Dictionary<ContainerRecipe, Port[]>(); var result = new List<ContainerRecipePortMapEntry>();
var ports = CreateServicePorts(containerRecipes); var ports = CreateServicePorts(containerRecipes);
@ -468,7 +468,7 @@ namespace KubernetesWorkflow
return (serviceSpec.Metadata.Name, result); return (serviceSpec.Metadata.Name, result);
} }
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result) private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
{ {
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s. // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace)); var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
@ -485,7 +485,8 @@ namespace KubernetesWorkflow
// These service ports belongs to this recipe. // These service ports belongs to this recipe.
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName)); var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray(); var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
result.Add(r, ports);
result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
} }
} }
} }

View File

@ -2,35 +2,50 @@
{ {
public class RunningPod public class RunningPod
{ {
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap; public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, ContainerRecipePortMapEntry[] portMapEntries)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
{ {
Cluster = cluster; Cluster = cluster;
PodInfo = podInfo; PodInfo = podInfo;
DeploymentName = deploymentName; DeploymentName = deploymentName;
ServiceName = serviceName; ServiceName = serviceName;
this.servicePortMap = servicePortMap; PortMapEntries = portMapEntries;
} }
public K8sCluster Cluster { get; } public K8sCluster Cluster { get; }
public PodInfo PodInfo { get; } public PodInfo PodInfo { get; }
public ContainerRecipePortMapEntry[] PortMapEntries { get; }
internal string DeploymentName { get; } internal string DeploymentName { get; }
internal string ServiceName { get; } internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{ {
if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>(); if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
return servicePortMap[containerRecipe]; {
return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
}
return Array.Empty<Port>();
} }
} }
public class ContainerRecipePortMapEntry
{
public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
{
ContainerNumber = containerNumber;
Ports = ports;
}
public int ContainerNumber { get; }
public Port[] Ports { get; }
}
public class PodInfo public class PodInfo
{ {
public PodInfo(string podName, string podIp, string k8sNodeName) public PodInfo(string name, string ip, string k8sNodeName)
{ {
Name = podName; Name = name;
Ip = podIp; Ip = ip;
K8SNodeName = k8sNodeName; K8SNodeName = k8sNodeName;
} }

View File

@ -64,7 +64,7 @@ namespace Tests.BasicTests
var contentId = buyer.UploadFile(testFile); var contentId = buyer.UploadFile(testFile);
buyer.Marketplace.RequestStorage(contentId, buyer.Marketplace.RequestStorage(contentId,
pricePerBytePerSecond: 2.TestTokens(), pricePerSlotPerSecond: 2.TestTokens(),
requiredCollateral: 10.TestTokens(), requiredCollateral: 10.TestTokens(),
minRequiredNumberOfNodes: 1, minRequiredNumberOfNodes: 1,
proofProbability: 5, proofProbability: 5,