codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup

This commit is contained in:
benbierens 2023-03-21 15:44:21 +01:00
parent 82e29d02c9
commit f3a5ed3976
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
5 changed files with 116 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
CodexDistTestCore/ActiveDeployment.cs
View File

@ -1,49 +0,0 @@
using k8s.Models;
namespace CodexDistTestCore
{
public class ActiveDeployment
{
public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers)
{
Origin = origin;
Containers = containers;
SelectorName = orderNumber.ToString().PadLeft(6, '0');
}
public OfflineCodexNodes Origin { get; }
public CodexNodeContainer[] Containers { get; }
public string SelectorName { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public List<string> ActivePodNames { get; } = new List<string>();
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + SelectorName,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + SelectorName,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
}
public string Describe()
{
return $"CodexNode{SelectorName}-{Origin.Describe()}";
}
}
}

121
CodexDistTestCore/K8sManager.cs
View File

@ -15,7 +15,7 @@ namespace CodexDistTestCore
public const string K8sNamespace = "codex-test-namespace";
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0);
private readonly Dictionary<OnlineCodexNodes, ActiveDeployment> activeDeployments = new Dictionary<OnlineCodexNodes, ActiveDeployment>();
private readonly List<OnlineCodexNodes> activeCodexNodes = new List<OnlineCodexNodes>();
private readonly List<string> knownActivePodNames = new List<string>();
private readonly IFileManager fileManager;
@ -24,37 +24,33 @@ namespace CodexDistTestCore
this.fileManager = fileManager;
}
public IOnlineCodexNodes BringOnline(OfflineCodexNodes node)
public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline)
{
var client = CreateClient();
EnsureTestNamespace(client);
var factory = new CodexNodeContainerFactory();
var list = new List<OnlineCodexNode>();
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < node.NumberOfNodes; i++)
{
var container = factory.CreateNext();
containers.Add(container);
var containers = CreateContainers(offline.NumberOfNodes);
var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online);
activeCodexNodes.Add(result);
var codexNode = new OnlineCodexNode(fileManager, container);
list.Add(codexNode);
}
CreateDeployment(client, result, offline);
CreateService(result, client);
var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray());
var result = new OnlineCodexNodes(this, list.ToArray());
activeDeployments.Add(result, activeDeployment);
CreateDeployment(activeDeployment, client, node);
CreateService(activeDeployment, client);
WaitUntilOnline(activeDeployment, client);
TestLog.Log($"{node.NumberOfNodes} Codex nodes online.");
WaitUntilOnline(result, client);
TestLog.Log($"{offline.NumberOfNodes} Codex nodes online.");
return result;
}
private CodexNodeContainer[] CreateContainers(int number)
{
var factory = new CodexNodeContainerFactory();
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < number; i++) containers.Add(factory.CreateNext());
return containers.ToArray();
}
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
{
var client = CreateClient();
@ -79,40 +75,40 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted(client);
}
public void FetchAllPodsLogs(Action<string, string, Stream> onLog)
public void FetchAllPodsLogs(Action<int, string, Stream> onLog)
{
var client = CreateClient();
foreach (var node in activeDeployments.Values)
foreach (var node in activeCodexNodes)
{
var nodeDescription = node.Describe();
foreach (var podName in node.ActivePodNames)
{
var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream);
onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream);
}
}
}
private void BringOffline(ActiveDeployment activeNode, Kubernetes client)
private void BringOffline(OnlineCodexNodes online, Kubernetes client)
{
DeleteDeployment(activeNode, client);
DeleteService(activeNode, client);
DeleteDeployment(client, online);
DeleteService(client, online);
}
#region Waiting
private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client)
private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client)
{
WaitUntil(() =>
{
activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace);
return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0;
online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
});
AssignActivePodNames(activeNode, client);
AssignActivePodNames(online, client);
}
private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client)
private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client)
{
var pods = client.ListNamespacedPod(K8sNamespace);
var podNames = pods.Items.Select(p => p.Name());
@ -121,7 +117,7 @@ namespace CodexDistTestCore
if (!knownActivePodNames.Contains(podName))
{
knownActivePodNames.Add(podName);
activeNode.ActivePodNames.Add(podName);
online.ActivePodNames.Add(podName);
}
}
}
@ -166,27 +162,28 @@ namespace CodexDistTestCore
#region Service management
private void CreateService(ActiveDeployment activeDeployment, Kubernetes client)
private void CreateService(OnlineCodexNodes online, Kubernetes client)
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = activeDeployment.GetServiceMetadata(),
Metadata = online.GetServiceMetadata(),
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = activeDeployment.GetSelector(),
Ports = CreateServicePorts(activeDeployment)
Selector = online.GetSelector(),
Ports = CreateServicePorts(online)
}
};
activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
}
private List<V1ServicePort> CreateServicePorts(ActiveDeployment activeDeployment)
private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
{
var result = new List<V1ServicePort>();
foreach (var container in activeDeployment.Containers)
var containers = online.GetContainers();
foreach (var container in containers)
{
result.Add(new V1ServicePort
{
@ -199,51 +196,52 @@ namespace CodexDistTestCore
return result;
}
private void DeleteService(ActiveDeployment node, Kubernetes client)
private void DeleteService(Kubernetes client, OnlineCodexNodes online)
{
if (node.Service == null) return;
client.DeleteNamespacedService(node.Service.Name(), K8sNamespace);
node.Service = null;
if (online.Service == null) return;
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
online.Service = null;
}
#endregion
#region Deployment management
private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode)
private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = node.GetDeploymentMetadata(),
Metadata = online.GetDeploymentMetadata(),
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = node.GetSelector()
MatchLabels = online.GetSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = node.GetSelector()
Labels = online.GetSelector()
},
Spec = new V1PodSpec
{
Containers = CreateDeploymentContainers(node, codexNode)
Containers = CreateDeploymentContainers(online, offline)
}
}
}
};
node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
}
private List<V1Container> CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode)
private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
{
var result = new List<V1Container>();
foreach (var container in node.Containers)
var containers = online.GetContainers();
foreach (var container in containers)
{
result.Add(new V1Container
{
@ -257,17 +255,17 @@ namespace CodexDistTestCore
Name = container.ContainerPortName
}
},
Env = dockerImage.CreateEnvironmentVariables(codexNode, container)
Env = dockerImage.CreateEnvironmentVariables(offline, container)
});
}
return result;
}
private void DeleteDeployment(ActiveDeployment node, Kubernetes client)
private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online)
{
if (node.Deployment == null) return;
client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace);
node.Deployment = null;
if (online.Deployment == null) return;
client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
online.Deployment = null;
}
#endregion
@ -312,12 +310,11 @@ namespace CodexDistTestCore
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
}
private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
{
var n = (OnlineCodexNodes)node;
var activeNode = activeDeployments[n];
activeDeployments.Remove(n);
return activeNode;
activeCodexNodes.Remove(n);
return n;
}
}
}

7
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -12,14 +12,15 @@ namespace CodexDistTestCore
public class OnlineCodexNode : IOnlineCodexNode
{
private readonly IFileManager fileManager;
private readonly CodexNodeContainer environment;
public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
{
this.fileManager = fileManager;
this.environment = environment;
Container = environment;
}
public CodexNodeContainer Container { get; }
public CodexDebugResponse GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
@ -47,7 +48,7 @@ namespace CodexDistTestCore
private Http Http()
{
return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1");
return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
}
}

53
CodexDistTestCore/OnlineCodexNodes.cs
View File

@ -1,4 +1,6 @@
namespace CodexDistTestCore
using k8s.Models;
namespace CodexDistTestCore
{
public interface IOnlineCodexNodes
{
@ -9,25 +11,66 @@
public class OnlineCodexNodes : IOnlineCodexNodes
{
private readonly IK8sManager k8SManager;
private readonly IOnlineCodexNode[] nodes;
public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes)
public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{
OrderNumber = orderNumber;
Origin = origin;
this.k8SManager = k8SManager;
this.nodes = nodes;
Nodes = nodes;
}
public IOnlineCodexNode this[int index]
{
get
{
return nodes[index];
return Nodes[index];
}
}
public int OrderNumber { get; }
public OfflineCodexNodes Origin { get; }
public OnlineCodexNode[] Nodes { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public List<string> ActivePodNames { get; } = new List<string>();
public IOfflineCodexNodes BringOffline()
{
return k8SManager.BringOffline(this);
}
public CodexNodeContainer[] GetContainers()
{
return Nodes.Select(n => n.Container).ToArray();
}
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + OrderNumber,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + OrderNumber,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };
}
public string Describe()
{
return $"CodexNode{OrderNumber}-{Origin.Describe()}";
}
}
}

9
CodexDistTestCore/TestLog.cs
View File

@ -62,15 +62,16 @@ namespace CodexDistTestCore
k8sManager.FetchAllPodsLogs(WritePodLog);
}
private static void WritePodLog(string id, string nodeDescription, Stream stream)
private static void WritePodLog(int id, string nodeDescription, Stream stream)
{
Log($"{nodeDescription} -->> {id}");
LogRaw(nodeDescription, id);
var logFile = id.ToString().PadLeft(6, '0');
Log($"{nodeDescription} -->> {logFile}");
LogRaw(nodeDescription, logFile);
var reader = new StreamReader(stream);
var line = reader.ReadLine();
while (line != null)
{
LogRaw(line, id);
LogRaw(line, logFile);
line = reader.ReadLine();
}
}