Cleanup of k8s manager
This commit is contained in:
benbierens
parent
f3a5ed3976
commit
a36e364996
|
|
@ -1,8 +1,4 @@
|
|||
using k8s;
|
||||
using k8s.Models;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IK8sManager
|
||||
{
|
||||
|
|
@ -12,11 +8,9 @@ namespace CodexDistTestCore
|
|||
|
||||
public class K8sManager : IK8sManager
|
||||
{
|
||||
public const string K8sNamespace = "codex-test-namespace";
|
||||
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
|
||||
private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0);
|
||||
private readonly List<OnlineCodexNodes> activeCodexNodes = new List<OnlineCodexNodes>();
|
||||
private readonly List<string> knownActivePodNames = new List<string>();
|
||||
private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0);
|
||||
private readonly List<OnlineCodexNodes> onlineCodexNodes = new List<OnlineCodexNodes>();
|
||||
private readonly KnownK8sPods knownPods = new KnownK8sPods();
|
||||
private readonly IFileManager fileManager;
|
||||
|
||||
public K8sManager(IFileManager fileManager)
|
||||
|
|
@ -26,20 +20,42 @@ namespace CodexDistTestCore
|
|||
|
||||
public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline)
|
||||
{
|
||||
var client = CreateClient();
|
||||
EnsureTestNamespace(client);
|
||||
var online = CreateOnlineCodexNodes(offline);
|
||||
|
||||
var containers = CreateContainers(offline.NumberOfNodes);
|
||||
var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
|
||||
var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online);
|
||||
activeCodexNodes.Add(result);
|
||||
K8s().BringOnline(online, offline);
|
||||
|
||||
CreateDeployment(client, result, offline);
|
||||
CreateService(result, client);
|
||||
|
||||
WaitUntilOnline(result, client);
|
||||
TestLog.Log($"{offline.NumberOfNodes} Codex nodes online.");
|
||||
|
||||
return online;
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
|
||||
{
|
||||
var online = GetAndRemoveActiveNodeFor(node);
|
||||
|
||||
K8s().BringOffline(online);
|
||||
|
||||
TestLog.Log($"{online.Describe()} offline.");
|
||||
|
||||
return online.Origin;
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
K8s().DeleteAllResources();
|
||||
}
|
||||
|
||||
public void FetchAllPodsLogs(IPodLogsHandler logHandler)
|
||||
{
|
||||
K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler);
|
||||
}
|
||||
|
||||
private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline)
|
||||
{
|
||||
var containers = CreateContainers(offline.NumberOfNodes);
|
||||
var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
|
||||
var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online);
|
||||
onlineCodexNodes.Add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -51,270 +67,16 @@ namespace CodexDistTestCore
|
|||
return containers.ToArray();
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
|
||||
{
|
||||
var client = CreateClient();
|
||||
|
||||
var activeNode = GetAndRemoveActiveNodeFor(node);
|
||||
|
||||
var deploymentName = activeNode.Deployment.Name();
|
||||
BringOffline(activeNode, client);
|
||||
WaitUntilOffline(deploymentName, client);
|
||||
TestLog.Log($"{activeNode.Describe()} offline.");
|
||||
|
||||
return activeNode.Origin;
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
var client = CreateClient();
|
||||
|
||||
DeleteNamespace(client);
|
||||
|
||||
WaitUntilZeroPods(client);
|
||||
WaitUntilNamespaceDeleted(client);
|
||||
}
|
||||
|
||||
public void FetchAllPodsLogs(Action<int, string, Stream> onLog)
|
||||
{
|
||||
var client = CreateClient();
|
||||
foreach (var node in activeCodexNodes)
|
||||
{
|
||||
var nodeDescription = node.Describe();
|
||||
foreach (var podName in node.ActivePodNames)
|
||||
{
|
||||
var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
|
||||
onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void BringOffline(OnlineCodexNodes online, Kubernetes client)
|
||||
{
|
||||
DeleteDeployment(client, online);
|
||||
DeleteService(client, online);
|
||||
}
|
||||
|
||||
#region Waiting
|
||||
|
||||
private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
|
||||
});
|
||||
|
||||
AssignActivePodNames(online, client);
|
||||
}
|
||||
|
||||
private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client)
|
||||
{
|
||||
var pods = client.ListNamespacedPod(K8sNamespace);
|
||||
var podNames = pods.Items.Select(p => p.Name());
|
||||
foreach (var podName in podNames)
|
||||
{
|
||||
if (!knownActivePodNames.Contains(podName))
|
||||
{
|
||||
knownActivePodNames.Add(podName);
|
||||
online.ActivePodNames.Add(podName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void WaitUntilOffline(string deploymentName, Kubernetes client)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
|
||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilZeroPods(Kubernetes client)
|
||||
{
|
||||
WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
|
||||
}
|
||||
|
||||
private void WaitUntilNamespaceDeleted(Kubernetes client)
|
||||
{
|
||||
WaitUntil(() => !IsTestNamespaceOnline(client));
|
||||
}
|
||||
|
||||
private void WaitUntil(Func<bool> predicate)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var state = predicate();
|
||||
while (!state)
|
||||
{
|
||||
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
|
||||
{
|
||||
Assert.Fail("K8s operation timed out.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Timing.WaitForK8sServiceDelay();
|
||||
state = predicate();
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Service management
|
||||
|
||||
private void CreateService(OnlineCodexNodes online, Kubernetes client)
|
||||
{
|
||||
var serviceSpec = new V1Service
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = online.GetServiceMetadata(),
|
||||
Spec = new V1ServiceSpec
|
||||
{
|
||||
Type = "NodePort",
|
||||
Selector = online.GetSelector(),
|
||||
Ports = CreateServicePorts(online)
|
||||
}
|
||||
};
|
||||
|
||||
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
|
||||
{
|
||||
var result = new List<V1ServicePort>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1ServicePort
|
||||
{
|
||||
Protocol = "TCP",
|
||||
Port = 8080,
|
||||
TargetPort = container.ContainerPortName,
|
||||
NodePort = container.ServicePort
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteService(Kubernetes client, OnlineCodexNodes online)
|
||||
{
|
||||
if (online.Service == null) return;
|
||||
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
|
||||
online.Service = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Deployment management
|
||||
|
||||
private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline)
|
||||
{
|
||||
var deploymentSpec = new V1Deployment
|
||||
{
|
||||
ApiVersion = "apps/v1",
|
||||
Metadata = online.GetDeploymentMetadata(),
|
||||
Spec = new V1DeploymentSpec
|
||||
{
|
||||
Replicas = 1,
|
||||
Selector = new V1LabelSelector
|
||||
{
|
||||
MatchLabels = online.GetSelector()
|
||||
},
|
||||
Template = new V1PodTemplateSpec
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Labels = online.GetSelector()
|
||||
},
|
||||
Spec = new V1PodSpec
|
||||
{
|
||||
Containers = CreateDeploymentContainers(online, offline)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
|
||||
{
|
||||
var result = new List<V1Container>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1Container
|
||||
{
|
||||
Name = container.Name,
|
||||
Image = dockerImage.GetImageTag(),
|
||||
Ports = new List<V1ContainerPort>
|
||||
{
|
||||
new V1ContainerPort
|
||||
{
|
||||
ContainerPort = container.ApiPort,
|
||||
Name = container.ContainerPortName
|
||||
}
|
||||
},
|
||||
Env = dockerImage.CreateEnvironmentVariables(offline, container)
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online)
|
||||
{
|
||||
if (online.Deployment == null) return;
|
||||
client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
online.Deployment = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Namespace management
|
||||
|
||||
private void EnsureTestNamespace(Kubernetes client)
|
||||
{
|
||||
if (IsTestNamespaceOnline(client)) return;
|
||||
|
||||
var namespaceSpec = new V1Namespace
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = K8sNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
||||
}
|
||||
};
|
||||
client.CreateNamespace(namespaceSpec);
|
||||
}
|
||||
|
||||
private void DeleteNamespace(Kubernetes client)
|
||||
{
|
||||
if (IsTestNamespaceOnline(client))
|
||||
{
|
||||
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private static Kubernetes CreateClient()
|
||||
{
|
||||
// todo: If the default KubeConfig file does not suffice, change it here:
|
||||
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
|
||||
return new Kubernetes(config);
|
||||
}
|
||||
|
||||
private static bool IsTestNamespaceOnline(Kubernetes client)
|
||||
{
|
||||
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
|
||||
}
|
||||
|
||||
private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
|
||||
{
|
||||
var n = (OnlineCodexNodes)node;
|
||||
activeCodexNodes.Remove(n);
|
||||
onlineCodexNodes.Remove(n);
|
||||
return n;
|
||||
}
|
||||
|
||||
private K8sOperations K8s()
|
||||
{
|
||||
return new K8sOperations(knownPods);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,270 @@
|
|||
using k8s;
|
||||
using k8s.Models;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public class K8sOperations
|
||||
{
|
||||
public const string K8sNamespace = "codex-test-namespace";
|
||||
|
||||
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
|
||||
private readonly Kubernetes client;
|
||||
private readonly KnownK8sPods knownPods;
|
||||
|
||||
public K8sOperations(KnownK8sPods knownPods)
|
||||
{
|
||||
this.knownPods = knownPods;
|
||||
|
||||
// todo: If the default KubeConfig file does not suffice, change it here:
|
||||
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
|
||||
client = new Kubernetes(config);
|
||||
}
|
||||
|
||||
public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline)
|
||||
{
|
||||
EnsureTestNamespace();
|
||||
|
||||
CreateDeployment(online, offline);
|
||||
CreateService(online);
|
||||
|
||||
WaitUntilOnline(online);
|
||||
AssignActivePodNames(online);
|
||||
}
|
||||
|
||||
public void BringOffline(OnlineCodexNodes online)
|
||||
{
|
||||
var deploymentName = online.Deployment.Name();
|
||||
DeleteDeployment(online);
|
||||
DeleteService(online);
|
||||
WaitUntilOffline(deploymentName);
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
DeleteNamespace();
|
||||
|
||||
WaitUntilZeroPods();
|
||||
WaitUntilNamespaceDeleted();
|
||||
}
|
||||
|
||||
public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler)
|
||||
{
|
||||
foreach (var online in onlines)
|
||||
{
|
||||
var nodeDescription = online.Describe();
|
||||
foreach (var podName in online.ActivePodNames)
|
||||
{
|
||||
var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
|
||||
logHandler.Log(online.OrderNumber, $"{nodeDescription}:{podName}", stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void AssignActivePodNames(OnlineCodexNodes online)
|
||||
{
|
||||
var pods = client.ListNamespacedPod(K8sNamespace);
|
||||
var podNames = pods.Items.Select(p => p.Name());
|
||||
foreach (var podName in podNames)
|
||||
{
|
||||
if (!knownPods.Contains(podName))
|
||||
{
|
||||
knownPods.Add(podName);
|
||||
online.ActivePodNames.Add(podName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#region Waiting
|
||||
|
||||
private void WaitUntilOnline(OnlineCodexNodes online)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilOffline(string deploymentName)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
|
||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilZeroPods()
|
||||
{
|
||||
WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
|
||||
}
|
||||
|
||||
private void WaitUntilNamespaceDeleted()
|
||||
{
|
||||
WaitUntil(() => !IsTestNamespaceOnline());
|
||||
}
|
||||
|
||||
private void WaitUntil(Func<bool> predicate)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var state = predicate();
|
||||
while (!state)
|
||||
{
|
||||
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
|
||||
{
|
||||
Assert.Fail("K8s operation timed out.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Timing.WaitForK8sServiceDelay();
|
||||
state = predicate();
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Service management
|
||||
|
||||
private void CreateService(OnlineCodexNodes online)
|
||||
{
|
||||
var serviceSpec = new V1Service
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = online.GetServiceMetadata(),
|
||||
Spec = new V1ServiceSpec
|
||||
{
|
||||
Type = "NodePort",
|
||||
Selector = online.GetSelector(),
|
||||
Ports = CreateServicePorts(online)
|
||||
}
|
||||
};
|
||||
|
||||
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
|
||||
{
|
||||
var result = new List<V1ServicePort>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1ServicePort
|
||||
{
|
||||
Protocol = "TCP",
|
||||
Port = 8080,
|
||||
TargetPort = container.ContainerPortName,
|
||||
NodePort = container.ServicePort
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteService(OnlineCodexNodes online)
|
||||
{
|
||||
if (online.Service == null) return;
|
||||
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
|
||||
online.Service = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Deployment management
|
||||
|
||||
private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline)
|
||||
{
|
||||
var deploymentSpec = new V1Deployment
|
||||
{
|
||||
ApiVersion = "apps/v1",
|
||||
Metadata = online.GetDeploymentMetadata(),
|
||||
Spec = new V1DeploymentSpec
|
||||
{
|
||||
Replicas = 1,
|
||||
Selector = new V1LabelSelector
|
||||
{
|
||||
MatchLabels = online.GetSelector()
|
||||
},
|
||||
Template = new V1PodTemplateSpec
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Labels = online.GetSelector()
|
||||
},
|
||||
Spec = new V1PodSpec
|
||||
{
|
||||
Containers = CreateDeploymentContainers(online, offline)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
|
||||
{
|
||||
var result = new List<V1Container>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1Container
|
||||
{
|
||||
Name = container.Name,
|
||||
Image = dockerImage.GetImageTag(),
|
||||
Ports = new List<V1ContainerPort>
|
||||
{
|
||||
new V1ContainerPort
|
||||
{
|
||||
ContainerPort = container.ApiPort,
|
||||
Name = container.ContainerPortName
|
||||
}
|
||||
},
|
||||
Env = dockerImage.CreateEnvironmentVariables(offline, container)
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteDeployment(OnlineCodexNodes online)
|
||||
{
|
||||
if (online.Deployment == null) return;
|
||||
client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
online.Deployment = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Namespace management
|
||||
|
||||
private void EnsureTestNamespace()
|
||||
{
|
||||
if (IsTestNamespaceOnline()) return;
|
||||
|
||||
var namespaceSpec = new V1Namespace
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = K8sNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
||||
}
|
||||
};
|
||||
client.CreateNamespace(namespaceSpec);
|
||||
}
|
||||
|
||||
private void DeleteNamespace()
|
||||
{
|
||||
if (IsTestNamespaceOnline())
|
||||
{
|
||||
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private bool IsTestNamespaceOnline()
|
||||
{
|
||||
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public class KnownK8sPods
|
||||
{
|
||||
private readonly List<string> knownActivePodNames = new List<string>();
|
||||
|
||||
public bool Contains(string name)
|
||||
{
|
||||
return knownActivePodNames.Contains(name);
|
||||
}
|
||||
|
||||
public void Add(string name)
|
||||
{
|
||||
knownActivePodNames.Add(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -28,6 +28,11 @@ namespace CodexDistTestCore
|
|||
}
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes BringOffline()
|
||||
{
|
||||
return k8SManager.BringOffline(this);
|
||||
}
|
||||
|
||||
public int OrderNumber { get; }
|
||||
public OfflineCodexNodes Origin { get; }
|
||||
public OnlineCodexNode[] Nodes { get; }
|
||||
|
|
@ -35,11 +40,6 @@ namespace CodexDistTestCore
|
|||
public V1Service? Service { get; set; }
|
||||
public List<string> ActivePodNames { get; } = new List<string>();
|
||||
|
||||
public IOfflineCodexNodes BringOffline()
|
||||
{
|
||||
return k8SManager.BringOffline(this);
|
||||
}
|
||||
|
||||
public CodexNodeContainer[] GetContainers()
|
||||
{
|
||||
return Nodes.Select(n => n.Container).ToArray();
|
||||
|
|
@ -50,7 +50,7 @@ namespace CodexDistTestCore
|
|||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-entrypoint-" + OrderNumber,
|
||||
NamespaceProperty = K8sManager.K8sNamespace
|
||||
NamespaceProperty = K8sOperations.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -59,7 +59,7 @@ namespace CodexDistTestCore
|
|||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-node-" + OrderNumber,
|
||||
NamespaceProperty = K8sManager.K8sNamespace
|
||||
NamespaceProperty = K8sOperations.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IPodLogsHandler
|
||||
{
|
||||
void Log(int id, string podDescription, Stream log);
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ namespace CodexDistTestCore
|
|||
|
||||
private static LogFile? file = null;
|
||||
|
||||
// This is all way too static. It needs to be cleaned up.
|
||||
public static void Log(string message)
|
||||
{
|
||||
file!.Write(message);
|
||||
|
|
@ -38,7 +39,8 @@ namespace CodexDistTestCore
|
|||
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
|
||||
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
||||
{
|
||||
IncludeFullPodLogging(k8sManager);
|
||||
var logWriter = new PodLogWriter(file);
|
||||
logWriter.IncludeFullPodLogging(k8sManager);
|
||||
}
|
||||
|
||||
file = null;
|
||||
|
|
@ -50,24 +52,29 @@ namespace CodexDistTestCore
|
|||
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
||||
return $"{className}.{test.MethodName}";
|
||||
}
|
||||
}
|
||||
|
||||
private static void LogRaw(string message, string filename)
|
||||
public class PodLogWriter : IPodLogsHandler
|
||||
{
|
||||
private readonly LogFile file;
|
||||
|
||||
public PodLogWriter(LogFile file)
|
||||
{
|
||||
file!.WriteRaw(message, filename);
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
private static void IncludeFullPodLogging(K8sManager k8sManager)
|
||||
public void IncludeFullPodLogging(K8sManager k8sManager)
|
||||
{
|
||||
Log("Full pod logging:");
|
||||
k8sManager.FetchAllPodsLogs(WritePodLog);
|
||||
TestLog.Log("Full pod logging:");
|
||||
k8sManager.FetchAllPodsLogs(this);
|
||||
}
|
||||
|
||||
private static void WritePodLog(int id, string nodeDescription, Stream stream)
|
||||
public void Log(int id, string podDescription, Stream log)
|
||||
{
|
||||
var logFile = id.ToString().PadLeft(6, '0');
|
||||
Log($"{nodeDescription} -->> {logFile}");
|
||||
LogRaw(nodeDescription, logFile);
|
||||
var reader = new StreamReader(stream);
|
||||
TestLog.Log($"{podDescription} -->> {logFile}");
|
||||
LogRaw(podDescription, logFile);
|
||||
var reader = new StreamReader(log);
|
||||
var line = reader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
|
|
@ -75,6 +82,11 @@ namespace CodexDistTestCore
|
|||
line = reader.ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
private void LogRaw(string message, string filename)
|
||||
{
|
||||
file!.WriteRaw(message, filename);
|
||||
}
|
||||
}
|
||||
|
||||
public class LogFile
|
||||
|
|
|
|||
Loading…
Reference in New Issue