codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup

This commit is contained in:
benbierens 2023-03-21 15:44:21 +01:00
parent 82e29d02c9
commit f3a5ed3976
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
5 changed files with 116 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
CodexDistTestCore/ActiveDeployment.cs
View File

@ -1,49 +0,0 @@
using k8s.Models;
namespace CodexDistTestCore
{
public class ActiveDeployment
{
public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers)
{
Origin = origin;
Containers = containers;
SelectorName = orderNumber.ToString().PadLeft(6, '0');
}
public OfflineCodexNodes Origin { get; }
public CodexNodeContainer[] Containers { get; }
public string SelectorName { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public List<string> ActivePodNames { get; } = new List<string>();
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + SelectorName,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + SelectorName,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
}
public string Describe()
{
return $"CodexNode{SelectorName}-{Origin.Describe()}";
}
}
}

121
CodexDistTestCore/K8sManager.cs
View File

@ -15,7 +15,7 @@ namespace CodexDistTestCore
public const string K8sNamespace = "codex-test-namespace"; public const string K8sNamespace = "codex-test-namespace";
private readonly CodexDockerImage dockerImage = new CodexDockerImage(); private readonly CodexDockerImage dockerImage = new CodexDockerImage();
private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0);
private readonly Dictionary<OnlineCodexNodes, ActiveDeployment> activeDeployments = new Dictionary<OnlineCodexNodes, ActiveDeployment>(); private readonly List<OnlineCodexNodes> activeCodexNodes = new List<OnlineCodexNodes>();
private readonly List<string> knownActivePodNames = new List<string>(); private readonly List<string> knownActivePodNames = new List<string>();
private readonly IFileManager fileManager; private readonly IFileManager fileManager;
@ -24,37 +24,33 @@ namespace CodexDistTestCore
this.fileManager = fileManager; this.fileManager = fileManager;
} }
public IOnlineCodexNodes BringOnline(OfflineCodexNodes node) public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline)
{ {
var client = CreateClient(); var client = CreateClient();
EnsureTestNamespace(client); EnsureTestNamespace(client);
var factory = new CodexNodeContainerFactory(); var containers = CreateContainers(offline.NumberOfNodes);
var list = new List<OnlineCodexNode>(); var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
var containers = new List<CodexNodeContainer>(); var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online);
for (var i = 0; i < node.NumberOfNodes; i++) activeCodexNodes.Add(result);
{
var container = factory.CreateNext();
containers.Add(container);
var codexNode = new OnlineCodexNode(fileManager, container); CreateDeployment(client, result, offline);
list.Add(codexNode); CreateService(result, client);
}
var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray()); WaitUntilOnline(result, client);
TestLog.Log($"{offline.NumberOfNodes} Codex nodes online.");
var result = new OnlineCodexNodes(this, list.ToArray());
activeDeployments.Add(result, activeDeployment);
CreateDeployment(activeDeployment, client, node);
CreateService(activeDeployment, client);
WaitUntilOnline(activeDeployment, client);
TestLog.Log($"{node.NumberOfNodes} Codex nodes online.");
return result; return result;
} }
private CodexNodeContainer[] CreateContainers(int number)
{
var factory = new CodexNodeContainerFactory();
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < number; i++) containers.Add(factory.CreateNext());
return containers.ToArray();
}
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
{ {
var client = CreateClient(); var client = CreateClient();
@ -79,40 +75,40 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted(client); WaitUntilNamespaceDeleted(client);
} }
public void FetchAllPodsLogs(Action<string, string, Stream> onLog) public void FetchAllPodsLogs(Action<int, string, Stream> onLog)
{ {
var client = CreateClient(); var client = CreateClient();
foreach (var node in activeDeployments.Values) foreach (var node in activeCodexNodes)
{ {
var nodeDescription = node.Describe(); var nodeDescription = node.Describe();
foreach (var podName in node.ActivePodNames) foreach (var podName in node.ActivePodNames)
{ {
var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream); onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream);
} }
} }
} }
private void BringOffline(ActiveDeployment activeNode, Kubernetes client) private void BringOffline(OnlineCodexNodes online, Kubernetes client)
{ {
DeleteDeployment(activeNode, client); DeleteDeployment(client, online);
DeleteService(activeNode, client); DeleteService(client, online);
} }
#region Waiting #region Waiting
private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client) private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client)
{ {
WaitUntil(() => WaitUntil(() =>
{ {
activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace); online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0; return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
}); });
AssignActivePodNames(activeNode, client); AssignActivePodNames(online, client);
} }
private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client) private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client)
{ {
var pods = client.ListNamespacedPod(K8sNamespace); var pods = client.ListNamespacedPod(K8sNamespace);
var podNames = pods.Items.Select(p => p.Name()); var podNames = pods.Items.Select(p => p.Name());
@ -121,7 +117,7 @@ namespace CodexDistTestCore
if (!knownActivePodNames.Contains(podName)) if (!knownActivePodNames.Contains(podName))
{ {
knownActivePodNames.Add(podName); knownActivePodNames.Add(podName);
activeNode.ActivePodNames.Add(podName); online.ActivePodNames.Add(podName);
} }
} }
} }
@ -166,27 +162,28 @@ namespace CodexDistTestCore
#region Service management #region Service management
private void CreateService(ActiveDeployment activeDeployment, Kubernetes client) private void CreateService(OnlineCodexNodes online, Kubernetes client)
{ {
var serviceSpec = new V1Service var serviceSpec = new V1Service
{ {
ApiVersion = "v1", ApiVersion = "v1",
Metadata = activeDeployment.GetServiceMetadata(), Metadata = online.GetServiceMetadata(),
Spec = new V1ServiceSpec Spec = new V1ServiceSpec
{ {
Type = "NodePort", Type = "NodePort",
Selector = activeDeployment.GetSelector(), Selector = online.GetSelector(),
Ports = CreateServicePorts(activeDeployment) Ports = CreateServicePorts(online)
} }
}; };
activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
} }
private List<V1ServicePort> CreateServicePorts(ActiveDeployment activeDeployment) private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
{ {
var result = new List<V1ServicePort>(); var result = new List<V1ServicePort>();
foreach (var container in activeDeployment.Containers) var containers = online.GetContainers();
foreach (var container in containers)
{ {
result.Add(new V1ServicePort result.Add(new V1ServicePort
{ {
@ -199,51 +196,52 @@ namespace CodexDistTestCore
return result; return result;
} }
private void DeleteService(ActiveDeployment node, Kubernetes client) private void DeleteService(Kubernetes client, OnlineCodexNodes online)
{ {
if (node.Service == null) return; if (online.Service == null) return;
client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
node.Service = null; online.Service = null;
} }
#endregion #endregion
#region Deployment management #region Deployment management
private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode) private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline)
{ {
var deploymentSpec = new V1Deployment var deploymentSpec = new V1Deployment
{ {
ApiVersion = "apps/v1", ApiVersion = "apps/v1",
Metadata = node.GetDeploymentMetadata(), Metadata = online.GetDeploymentMetadata(),
Spec = new V1DeploymentSpec Spec = new V1DeploymentSpec
{ {
Replicas = 1, Replicas = 1,
Selector = new V1LabelSelector Selector = new V1LabelSelector
{ {
MatchLabels = node.GetSelector() MatchLabels = online.GetSelector()
}, },
Template = new V1PodTemplateSpec Template = new V1PodTemplateSpec
{ {
Metadata = new V1ObjectMeta Metadata = new V1ObjectMeta
{ {
Labels = node.GetSelector() Labels = online.GetSelector()
}, },
Spec = new V1PodSpec Spec = new V1PodSpec
{ {
Containers = CreateDeploymentContainers(node, codexNode) Containers = CreateDeploymentContainers(online, offline)
} }
} }
} }
}; };
node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
} }
private List<V1Container> CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode) private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
{ {
var result = new List<V1Container>(); var result = new List<V1Container>();
foreach (var container in node.Containers) var containers = online.GetContainers();
foreach (var container in containers)
{ {
result.Add(new V1Container result.Add(new V1Container
{ {
@ -257,17 +255,17 @@ namespace CodexDistTestCore
Name = container.ContainerPortName Name = container.ContainerPortName
} }
}, },
Env = dockerImage.CreateEnvironmentVariables(codexNode, container) Env = dockerImage.CreateEnvironmentVariables(offline, container)
}); });
} }
return result; return result;
} }
private void DeleteDeployment(ActiveDeployment node, Kubernetes client) private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online)
{ {
if (node.Deployment == null) return; if (online.Deployment == null) return;
client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
node.Deployment = null; online.Deployment = null;
} }
#endregion #endregion
@ -312,12 +310,11 @@ namespace CodexDistTestCore
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
} }
private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
{ {
var n = (OnlineCodexNodes)node; var n = (OnlineCodexNodes)node;
var activeNode = activeDeployments[n]; activeCodexNodes.Remove(n);
activeDeployments.Remove(n); return n;
return activeNode;
} }
} }
} }

7
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -12,14 +12,15 @@ namespace CodexDistTestCore
public class OnlineCodexNode : IOnlineCodexNode public class OnlineCodexNode : IOnlineCodexNode
{ {
private readonly IFileManager fileManager; private readonly IFileManager fileManager;
private readonly CodexNodeContainer environment;
public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
{ {
this.fileManager = fileManager; this.fileManager = fileManager;
this.environment = environment; Container = environment;
} }
public CodexNodeContainer Container { get; }
public CodexDebugResponse GetDebugInfo() public CodexDebugResponse GetDebugInfo()
{ {
return Http().HttpGetJson<CodexDebugResponse>("debug/info"); return Http().HttpGetJson<CodexDebugResponse>("debug/info");
@ -47,7 +48,7 @@ namespace CodexDistTestCore
private Http Http() private Http Http()
{ {
return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1"); return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
} }
} }

53
CodexDistTestCore/OnlineCodexNodes.cs
View File

@ -1,4 +1,6 @@
namespace CodexDistTestCore using k8s.Models;
namespace CodexDistTestCore
{ {
public interface IOnlineCodexNodes public interface IOnlineCodexNodes
{ {
@ -9,25 +11,66 @@
public class OnlineCodexNodes : IOnlineCodexNodes public class OnlineCodexNodes : IOnlineCodexNodes
{ {
private readonly IK8sManager k8SManager; private readonly IK8sManager k8SManager;
private readonly IOnlineCodexNode[] nodes;
public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes) public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{ {
OrderNumber = orderNumber;
Origin = origin;
this.k8SManager = k8SManager; this.k8SManager = k8SManager;
this.nodes = nodes; Nodes = nodes;
} }
public IOnlineCodexNode this[int index] public IOnlineCodexNode this[int index]
{ {
get get
{ {
return nodes[index]; return Nodes[index];
} }
} }
public int OrderNumber { get; }
public OfflineCodexNodes Origin { get; }
public OnlineCodexNode[] Nodes { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public List<string> ActivePodNames { get; } = new List<string>();
public IOfflineCodexNodes BringOffline() public IOfflineCodexNodes BringOffline()
{ {
return k8SManager.BringOffline(this); return k8SManager.BringOffline(this);
} }
public CodexNodeContainer[] GetContainers()
{
return Nodes.Select(n => n.Container).ToArray();
}
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + OrderNumber,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + OrderNumber,
NamespaceProperty = K8sManager.K8sNamespace
};
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };
}
public string Describe()
{
return $"CodexNode{OrderNumber}-{Origin.Describe()}";
}
} }
} }

9
CodexDistTestCore/TestLog.cs
View File

@ -62,15 +62,16 @@ namespace CodexDistTestCore
k8sManager.FetchAllPodsLogs(WritePodLog); k8sManager.FetchAllPodsLogs(WritePodLog);
} }
private static void WritePodLog(string id, string nodeDescription, Stream stream) private static void WritePodLog(int id, string nodeDescription, Stream stream)
{ {
Log($"{nodeDescription} -->> {id}"); var logFile = id.ToString().PadLeft(6, '0');
LogRaw(nodeDescription, id); Log($"{nodeDescription} -->> {logFile}");
LogRaw(nodeDescription, logFile);
var reader = new StreamReader(stream); var reader = new StreamReader(stream);
var line = reader.ReadLine(); var line = reader.ReadLine();
while (line != null) while (line != null)
{ {
LogRaw(line, id); LogRaw(line, logFile);
line = reader.ReadLine(); line = reader.ReadLine();
} }
} }