status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automatically map location enum to available k8s nodes.

This commit is contained in:
benbierens 2023-06-02 10:04:07 +02:00
parent e7d059ceed
commit ad71cff465
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
16 changed files with 81 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -49,7 +49,7 @@ namespace DistTestCore.Codex
var companionNodeAccount = companionNode.Accounts[Index];
Additional(companionNodeAccount);
var ip = companionNode.RunningContainer.Pod.Ip;
var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");

3
DistTestCore/CodexStarter.cs
View File

@ -28,7 +28,8 @@ namespace DistTestCore
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
var podInfo = group.Containers.RunningPod.PodInfo;
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
LogSeparator();
return group;
}

7
DistTestCore/Configuration.cs
View File

@ -28,12 +28,7 @@ namespace DistTestCore
k8sNamespacePrefix: "ct-",
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
}
retryDelay: timeSet.WaitForK8sServiceDelay()
);
}

2
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -138,7 +138,7 @@ namespace DistTestCore.Helpers
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.Ip;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}

2
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
{
var startupConfig = new StartupConfig();
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
startupConfig.Add(contractsConfig);
return startupConfig;
}

2
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
var httpPort = AddExposedPort(tag: HttpPortTag);
var bootPubKey = config.BootstrapNode.PubKey;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
var bootPort = config.BootstrapNode.DiscoveryPort.Number;
var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";

2
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
private string GetInstanceNameForNode(RunningContainer node)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
return $"{ip}:{port}";
}

2
DistTestCore/OnlineCodexNode.cs
View File

@ -124,7 +124,7 @@ namespace DistTestCore
// The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress.
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
}
private void DownloadToFile(string contentId, TestFile file)

2
DistTestCore/PrometheusStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
foreach (var node in nodes)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
config += $" - '{ip}:{port}'\n";
}

2
KubernetesWorkflow/CommandRunner.cs
View File

@ -28,7 +28,7 @@ namespace KubernetesWorkflow
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

16
KubernetesWorkflow/Configuration.cs
View File

@ -2,31 +2,17 @@
{
public class Configuration
{
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
{
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }
public ConfigurationLocationEntry[] LocationMap { get; }
}
public class ConfigurationLocationEntry
{
public ConfigurationLocationEntry(Location location, string workerName)
{
Location = location;
WorkerName = workerName;
}
public Location Location { get; }
public string WorkerName { get; }
}
}

19
KubernetesWorkflow/K8sCluster.cs
View File

@ -11,6 +11,7 @@ namespace KubernetesWorkflow
public Configuration Configuration { get; }
public string HostAddress { get; private set; } = string.Empty;
public string[] AvailableK8sNodes { get; set; } = new string[0];
public KubernetesClientConfiguration GetK8sClientConfig()
{
@ -21,8 +22,16 @@ namespace KubernetesWorkflow
public string GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
switch (location)
{
case Location.One:
return K8sNodeIfAvailable(0);
case Location.Two:
return K8sNodeIfAvailable(1);
case Location.Three:
return K8sNodeIfAvailable(2);
}
return string.Empty;
}
public TimeSpan K8sOperationTimeout()
@ -59,5 +68,11 @@ namespace KubernetesWorkflow
HostAddress = config.Host;
}
}
private string K8sNodeIfAvailable(int index)
{
if (AvailableK8sNodes.Length <= index) return string.Empty;
return AvailableK8sNodes[index];
}
}
}

36
KubernetesWorkflow/K8sController.cs
View File

@ -32,13 +32,14 @@ namespace KubernetesWorkflow
public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
{
log.Debug();
DiscoverK8sNodes();
EnsureTestNamespace();
var deploymentName = CreateDeployment(containerRecipes, location);
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var (podName, podIp) = FetchNewPod();
var podInfo = FetchNewPod();
return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
}
public void Stop(RunningPod pod)
@ -47,13 +48,13 @@ namespace KubernetesWorkflow
if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
DeleteDeployment(pod.DeploymentName);
WaitUntilDeploymentOffline(pod.DeploymentName);
WaitUntilPodOffline(pod.Name);
WaitUntilPodOffline(pod.PodInfo.Name);
}
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
@ -106,6 +107,28 @@ namespace KubernetesWorkflow
}
}
#region Discover K8s Nodes
private void DiscoverK8sNodes()
{
if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
{
cluster.AvailableK8sNodes = GetAvailableK8sNodes();
if (cluster.AvailableK8sNodes.Length < 3)
{
log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes)}'.");
}
}
}
private string[] GetAvailableK8sNodes()
{
var nodes = client.Run(c => c.ListNode());
return nodes.Items.Select(i => i.Metadata.Name).ToArray();
}
#endregion
#region Namespace management
private string K8sTestNamespace { get; }
@ -537,7 +560,7 @@ namespace KubernetesWorkflow
#endregion
private (string, string) FetchNewPod()
private PodInfo FetchNewPod()
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
@ -547,12 +570,13 @@ namespace KubernetesWorkflow
var newPod = newPods.Single();
var name = newPod.Name();
var ip = newPod.Status.PodIP;
var k8sNodeName = newPod.Spec.NodeName;
if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
knownPods.Add(name);
return (name, ip);
return new PodInfo(name, ip, k8sNodeName);
}
}
}

5
KubernetesWorkflow/Location.cs
View File

@ -3,7 +3,8 @@
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine
One,
Two,
Three,
}
}

22
KubernetesWorkflow/RunningPod.cs
View File

@ -4,19 +4,17 @@
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
{
Cluster = cluster;
Name = name;
Ip = ip;
PodInfo = podInfo;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
}
public K8sCluster Cluster { get; }
public string Name { get; }
public string Ip { get; }
public PodInfo PodInfo { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
@ -25,4 +23,18 @@
return servicePortMap[containerRecipe];
}
}
public class PodInfo
{
public PodInfo(string podName, string podIp, string k8sNodeName)
{
Name = podName;
Ip = podIp;
K8SNodeName = k8sNodeName;
}
public string Name { get; }
public string Ip { get; }
public string K8SNodeName { get; }
}
}

5
Tests/BasicTests/TwoClientTests.cs
View File

@ -28,11 +28,10 @@ namespace Tests.BasicTests
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
var primary = SetupCodexNode(s => s.At(Location.One));
var secondary = SetupCodexNode(s => s.At(Location.Two));
PerformTwoClientTest(primary, secondary);
}