vacp2p
/
cs-codex-dist-tests
mirror of https://github.com/vacp2p/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/configurability'

This commit is contained in:
benbierens 2023-06-04 07:56:50 +02:00
parent e8d99b83ff 7c4a2ea12c
commit bc4035e723
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
20 changed files with 201 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -49,7 +49,7 @@ namespace DistTestCore.Codex
var companionNodeAccount = companionNode.Accounts[Index];
Additional(companionNodeAccount);
var ip = companionNode.RunningContainer.Pod.Ip;
var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");

3
DistTestCore/CodexStarter.cs
View File

@ -28,7 +28,8 @@ namespace DistTestCore
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
var podInfo = group.Containers.RunningPod.PodInfo;
LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
LogSeparator();
return group;
}

53
DistTestCore/Configuration.cs
View File

@ -5,39 +5,51 @@ namespace DistTestCore
{
public class Configuration
{
private readonly string? kubeConfigFile;
private readonly string logPath;
private readonly bool logDebug;
private readonly string dataFilesPath;
private readonly CodexLogLevel codexLogLevel;
private readonly TestRunnerLocation runnerLocation;
public Configuration()
{
kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
}
retryDelay: timeSet.WaitForK8sServiceDelay()
);
}
public Logging.LogConfig GetLogConfig()
{
return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
return new Logging.LogConfig(logPath, debugEnabled: logDebug);
}
public string GetFileManagerFolder()
{
return "TestDataFiles";
return dataFilesPath;
}
public CodexLogLevel GetCodexLogLevel()
{
return CodexLogLevel.Trace;
return codexLogLevel;
}
public TestRunnerLocation GetTestRunnerLocation()
{
return TestRunnerLocation.ExternalToCluster;
return runnerLocation;
}
public RunningContainerAddress GetAddress(RunningContainer container)
@ -48,6 +60,25 @@ namespace DistTestCore
}
return container.ClusterExternalAddress;
}
private static string GetEnvVarOrDefault(string varName, string defaultValue)
{
var v = Environment.GetEnvironmentVariable(varName);
if (v == null) return defaultValue;
return v;
}
private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
{
var v = Environment.GetEnvironmentVariable(varName);
if (v == null) return defaultValue;
return v;
}
private static T ParseEnum<T>(string value)
{
return (T)Enum.Parse(typeof(T), value, true);
}
}
public enum TestRunnerLocation

2
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -138,7 +138,7 @@ namespace DistTestCore.Helpers
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.Ip;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}

2
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
{
var startupConfig = new StartupConfig();
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
startupConfig.Add(contractsConfig);
return startupConfig;
}

2
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
var httpPort = AddExposedPort(tag: HttpPortTag);
var bootPubKey = config.BootstrapNode.PubKey;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
var bootPort = config.BootstrapNode.DiscoveryPort.Number;
var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";

2
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
private string GetInstanceNameForNode(RunningContainer node)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
return $"{ip}:{port}";
}

2
DistTestCore/OnlineCodexNode.cs
View File

@ -124,7 +124,7 @@ namespace DistTestCore
// The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress.
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
}
private void DownloadToFile(string contentId, TestFile file)

2
DistTestCore/PrometheusStarter.cs
View File

@ -44,7 +44,7 @@ namespace DistTestCore
foreach (var node in nodes)
{
var ip = node.Pod.Ip;
var ip = node.Pod.PodInfo.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
config += $" - '{ip}:{port}'\n";
}

40
KubernetesWorkflow/ApplicationLifecycle.cs
View File

@ -1,40 +0,0 @@
using Utils;
namespace KubernetesWorkflow
{
public class ApplicationLifecycle
{
private static object instanceLock = new object();
private static ApplicationLifecycle? instance;
private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
private readonly NumberSource namespaceNumberSource = new NumberSource(0);
private ApplicationLifecycle()
{
}
public static ApplicationLifecycle Instance
{
// I know singletons are quite evil. But we need to be sure this object is created only once
// and persists for the entire application lifecycle.
get
{
lock (instanceLock)
{
if (instance == null) instance = new ApplicationLifecycle();
return instance;
}
}
}
public NumberSource GetServiceNumberSource()
{
return servicePortNumberSource;
}
public string GetTestNamespace()
{
return namespaceNumberSource.GetNextNumber().ToString("D5");
}
}
}

2
KubernetesWorkflow/CommandRunner.cs
View File

@ -28,7 +28,7 @@ namespace KubernetesWorkflow
var input = new[] { command }.Concat(arguments).ToArray();
Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
}
public string GetStdOut()

16
KubernetesWorkflow/Configuration.cs
View File

@ -2,31 +2,17 @@
{
public class Configuration
{
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
{
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
LocationMap = locationMap;
}
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }
public ConfigurationLocationEntry[] LocationMap { get; }
}
public class ConfigurationLocationEntry
{
public ConfigurationLocationEntry(Location location, string workerName)
{
Location = location;
WorkerName = workerName;
}
public Location Location { get; }
public string WorkerName { get; }
}
}

33
KubernetesWorkflow/K8sCluster.cs
View File

@ -11,6 +11,7 @@ namespace KubernetesWorkflow
public Configuration Configuration { get; }
public string HostAddress { get; private set; } = string.Empty;
public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];
public KubernetesClientConfiguration GetK8sClientConfig()
{
@ -19,10 +20,18 @@ namespace KubernetesWorkflow
return config;
}
public string GetNodeLabelForLocation(Location location)
public K8sNodeLabel? GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
switch (location)
{
case Location.One:
return K8sNodeIfAvailable(0);
case Location.Two:
return K8sNodeIfAvailable(1);
case Location.Three:
return K8sNodeIfAvailable(2);
}
return null;
}
public TimeSpan K8sOperationTimeout()
@ -59,5 +68,23 @@ namespace KubernetesWorkflow
HostAddress = config.Host;
}
}
private K8sNodeLabel? K8sNodeIfAvailable(int index)
{
if (AvailableK8sNodes.Length <= index) return null;
return AvailableK8sNodes[index];
}
}
public class K8sNodeLabel
{
public K8sNodeLabel(string key, string value)
{
Key = key;
Value = value;
}
public string Key { get; }
public string Value { get; }
}
}

100
KubernetesWorkflow/K8sController.cs
View File

@ -32,13 +32,14 @@ namespace KubernetesWorkflow
public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
{
log.Debug();
DiscoverK8sNodes();
EnsureTestNamespace();
var deploymentName = CreateDeployment(containerRecipes, location);
var (serviceName, servicePortsMap) = CreateService(containerRecipes);
var (podName, podIp) = FetchNewPod();
var podInfo = FetchNewPod();
return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
}
public void Stop(RunningPod pod)
@ -47,13 +48,13 @@ namespace KubernetesWorkflow
if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
DeleteDeployment(pod.DeploymentName);
WaitUntilDeploymentOffline(pod.DeploymentName);
WaitUntilPodOffline(pod.Name);
WaitUntilPodOffline(pod.PodInfo.Name);
}
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
{
log.Debug();
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
logHandler.Log(stream);
}
@ -106,6 +107,42 @@ namespace KubernetesWorkflow
}
}
#region Discover K8s Nodes
private void DiscoverK8sNodes()
{
if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
{
cluster.AvailableK8sNodes = GetAvailableK8sNodes();
if (cluster.AvailableK8sNodes.Length < 3)
{
log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes.Select(p => $"{p.Key}={p.Value}"))}'.");
}
}
}
private K8sNodeLabel[] GetAvailableK8sNodes()
{
var nodes = client.Run(c => c.ListNode());
var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
return optionals.Where(n => n != null).Select(n => n!).ToArray();
}
private K8sNodeLabel? CreateNodeLabel(V1Node i)
{
var keys = i.Metadata.Labels.Keys;
var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
if (hostnameKey != null)
{
var hostnameValue = i.Metadata.Labels[hostnameKey];
return new K8sNodeLabel(hostnameKey, hostnameValue);
}
return null;
}
#endregion
#region Namespace management
private string K8sTestNamespace { get; }
@ -314,11 +351,12 @@ namespace KubernetesWorkflow
private IDictionary<string, string> CreateNodeSelector(Location location)
{
if (location == Location.Unspecified) return new Dictionary<string, string>();
var nodeLabel = cluster.GetNodeLabelForLocation(location);
if (nodeLabel == null) return new Dictionary<string, string>();
return new Dictionary<string, string>
{
{ "codex-test-location", cluster.GetNodeLabelForLocation(location) }
{ nodeLabel.Key, nodeLabel.Value }
};
}
@ -401,7 +439,7 @@ namespace KubernetesWorkflow
{
var result = new Dictionary<ContainerRecipe, Port[]>();
var ports = CreateServicePorts(result, containerRecipes);
var ports = CreateServicePorts(containerRecipes);
if (!ports.Any())
{
@ -424,9 +462,40 @@ namespace KubernetesWorkflow
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
return (serviceSpec.Metadata.Name, result);
}
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
{
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
foreach (var r in containerRecipes)
{
if (r.ExposedPorts.Any())
{
var firstExposedPort = r.ExposedPorts.First();
var portName = GetNameForPort(r, firstExposedPort);
var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
if (matchingServicePorts.Any())
{
// These service ports belongs to this recipe.
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
result.Add(r, ports);
}
}
}
}
private Port? MapNodePortIfAble(V1ServicePort p, string tag)
{
if (p.NodePort == null) return null;
return new Port(p.NodePort.Value, tag);
}
private void DeleteService(string serviceName)
{
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
@ -441,36 +510,30 @@ namespace KubernetesWorkflow
};
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe[] recipes)
private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
{
var result = new List<V1ServicePort>();
foreach (var recipe in recipes)
{
result.AddRange(CreateServicePorts(servicePorts, recipe));
result.AddRange(CreateServicePorts(recipe));
}
return result;
}
private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe recipe)
private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
{
var result = new List<V1ServicePort>();
var usedPorts = new List<Port>();
foreach (var port in recipe.ExposedPorts)
{
var servicePort = workflowNumberSource.GetServicePort();
usedPorts.Add(new Port(servicePort, ""));
result.Add(new V1ServicePort
{
Name = GetNameForPort(recipe, port),
Protocol = "TCP",
Port = port.Number,
TargetPort = GetNameForPort(recipe, port),
NodePort = servicePort
});
}
servicePorts.Add(recipe, usedPorts.ToArray());
return result;
}
@ -537,7 +600,7 @@ namespace KubernetesWorkflow
#endregion
private (string, string) FetchNewPod()
private PodInfo FetchNewPod()
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
@ -547,12 +610,13 @@ namespace KubernetesWorkflow
var newPod = newPods.Single();
var name = newPod.Name();
var ip = newPod.Status.PodIP;
var k8sNodeName = newPod.Spec.NodeName;
if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
knownPods.Add(name);
return (name, ip);
return new PodInfo(name, ip, k8sNodeName);
}
}
}

5
KubernetesWorkflow/Location.cs
View File

@ -3,7 +3,8 @@
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine
One,
Two,
Three,
}
}

23
KubernetesWorkflow/RunningPod.cs
View File

@ -4,25 +4,38 @@
{
private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
{
Cluster = cluster;
Name = name;
Ip = ip;
PodInfo = podInfo;
DeploymentName = deploymentName;
ServiceName = serviceName;
this.servicePortMap = servicePortMap;
}
public K8sCluster Cluster { get; }
public string Name { get; }
public string Ip { get; }
public PodInfo PodInfo { get; }
internal string DeploymentName { get; }
internal string ServiceName { get; }
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
{
if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
return servicePortMap[containerRecipe];
}
}
public class PodInfo
{
public PodInfo(string podName, string podIp, string k8sNodeName)
{
Name = podName;
Ip = podIp;
K8SNodeName = k8sNodeName;
}
public string Name { get; }
public string Ip { get; }
public string K8SNodeName { get; }
}
}

3
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -16,13 +16,12 @@ namespace KubernetesWorkflow
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
}
public StartupWorkflow CreateWorkflow()
{
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);

9
KubernetesWorkflow/WorkflowNumberSource.cs
View File

@ -4,13 +4,11 @@ namespace KubernetesWorkflow
{
public class WorkflowNumberSource
{
private readonly NumberSource servicePortNumberSource;
private readonly NumberSource containerNumberSource;
public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource)
{
WorkflowNumber = workflowNumber;
this.servicePortNumberSource = servicePortNumberSource;
this.containerNumberSource = containerNumberSource;
}
@ -20,10 +18,5 @@ namespace KubernetesWorkflow
{
return containerNumberSource.GetNextNumber();
}
public int GetServicePort()
{
return servicePortNumberSource.GetNextNumber();
}
}
}

11
README.md
View File

@ -17,6 +17,17 @@ Tests are devided into two assemblies: `/Tests` and `/LongTests`.
TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally.
## Configuration
Test executing can be configured using the following environment variables.
| Variable | Description | Default |
|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
| KUBECONFIG | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available. | (null) |
| LOGPATH | Path (abs or rel) where log files will be saved. | "CodexTestLogs" |
| LOGDEBUG | When "true", enables additional test-runner debug log output. | "false" |
| DATAFILEPATH | Path (abs or rel) where temporary test data files will be saved. | "TestDataFiles" |
| LOGLEVEL | Codex log-level. (case-insensitive) | "Trace" |
| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
## Test logs
Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.

5
Tests/BasicTests/TwoClientTests.cs
View File

@ -28,11 +28,10 @@ namespace Tests.BasicTests
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
var primary = SetupCodexNode(s => s.At(Location.One));
var secondary = SetupCodexNode(s => s.At(Location.Two));
PerformTwoClientTest(primary, secondary);
}