vacp2p
/
cs-codex-dist-tests
mirror of https://github.com/vacp2p/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleaning up

This commit is contained in:
benbierens 2023-03-19 11:40:05 +01:00
parent 686cdc88e2
commit 63d1d9b451
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
3 changed files with 67 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
TestCore/K8sManager.cs
View File

@ -1,11 +1,13 @@
using k8s;
using k8s.Models;
using NUnit.Framework;
namespace CodexDistTests.TestCore
{
public interface IK8sManager
{
IOnlineCodexNode BringOnline(OfflineCodexNode node);
IOfflineCodexNode BringOffline(IOnlineCodexNode node);
}
public class K8sManager : IK8sManager
@ -32,8 +34,8 @@ namespace CodexDistTests.TestCore
EnsureTestNamespace(client);
var activeNode = new ActiveNode(GetFreePort(), GetNodeOrderNumber());
var codexNode = new OnlineCodexNode(node, fileManager, activeNode.Port);
var activeNode = new ActiveNode(node, GetFreePort(), GetNodeOrderNumber());
var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port);
activeNodes.Add(codexNode, activeNode);
CreateDeployment(activeNode, client, node);
@ -48,15 +50,13 @@ namespace CodexDistTests.TestCore
{
var client = CreateClient();
var n = (OnlineCodexNode)node;
var activeNode = activeNodes[n];
activeNodes.Remove(n);
var activeNode = GetAndRemoveActiveNodeFor(node);
var deploymentName = activeNode.Deployment.Name();
BringOffline(activeNode, client);
WaitUntilOffline(deploymentName, client);
return n.Origin;
return activeNode.Origin;
}
public void DeleteAllResources()
@ -84,40 +84,46 @@ namespace CodexDistTests.TestCore
private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
{
while (activeNode.Deployment?.Status.AvailableReplicas == null || activeNode.Deployment.Status.AvailableReplicas != 1)
{
Timing.WaitForServiceDelay();
activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace);
}
WaitUntil(() =>
activeNode.Deployment?.Status.AvailableReplicas != null &&
activeNode.Deployment.Status.AvailableReplicas > 0);
}
private void WaitUntilOffline(string deploymentName, Kubernetes client)
{
var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
while (deployment != null && deployment.Status.AvailableReplicas > 0)
WaitUntil(() =>
{
Timing.WaitForServiceDelay();
deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
}
var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
}
private void WaitUntilZeroPods(Kubernetes client)
{
var pods = client.ListNamespacedPod(k8sNamespace);
while (pods.Items.Any())
{
Timing.WaitForServiceDelay();
pods = client.ListNamespacedPod(k8sNamespace);
}
WaitUntil(() =>
!client.ListNamespacedPod(k8sNamespace).Items.Any());
}
private void WaitUntilNamespaceDeleted(Kubernetes client)
{
var namespaces = client.ListNamespace();
while (namespaces.Items.Any(n => n.Metadata.Name == k8sNamespace))
WaitUntil(() =>
client.ListNamespace().Items.All(n => n.Metadata.Name != k8sNamespace));
}
private void WaitUntil(Func<bool> predicate)
{
Timing.WaitForServiceDelay();
namespaces = client.ListNamespace();
var start = DateTime.UtcNow;
var state = predicate();
while (!state)
{
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
{
Assert.Fail("K8s operation timed out.");
throw new TimeoutException();
}
Timing.WaitForK8sServiceDelay();
state = predicate();
}
}
@ -250,6 +256,14 @@ namespace CodexDistTests.TestCore
return new Kubernetes(config);
}
private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
{
var n = (OnlineCodexNode)node;
var activeNode = activeNodes[n];
activeNodes.Remove(n);
return activeNode;
}
private int GetFreePort()
{
var port = freePort;
@ -266,12 +280,14 @@ namespace CodexDistTests.TestCore
public class ActiveNode
{
public ActiveNode(int port, int orderNumber)
public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
{
Origin = origin;
SelectorName = orderNumber.ToString().PadLeft(6, '0');
Port = port;
}
public OfflineCodexNode Origin { get; }
public string SelectorName { get; }
public int Port { get; }
public V1Deployment? Deployment { get; set; }

19
TestCore/OnlineCodexNode.cs
View File

@ -9,22 +9,27 @@ namespace CodexDistTests.TestCore
CodexDebugResponse GetDebugInfo();
ContentId UploadFile(TestFile file, int retryCounter = 0);
TestFile? DownloadContent(ContentId contentId);
IOfflineCodexNode BringOffline();
}
public class OnlineCodexNode : IOnlineCodexNode
{
private readonly IK8sManager k8SManager;
private readonly IFileManager fileManager;
private readonly int port;
public OfflineCodexNode Origin { get; }
public OnlineCodexNode(OfflineCodexNode origin, IFileManager fileManager, int port)
public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port)
{
Origin = origin;
this.k8SManager = k8SManager;
this.fileManager = fileManager;
this.port = port;
}
public IOfflineCodexNode BringOffline()
{
return k8SManager.BringOffline(this);
}
public CodexDebugResponse GetDebugInfo()
{
return HttpGet<CodexDebugResponse>("debug/info");
@ -48,7 +53,7 @@ namespace CodexDistTests.TestCore
}
catch (Exception exception)
{
if (retryCounter > 5)
if (retryCounter > Timing.HttpCallRetryCount())
{
Assert.Fail(exception.Message);
throw;
@ -83,7 +88,7 @@ namespace CodexDistTests.TestCore
}
catch (Exception exception)
{
if (retryCounter > 5)
if (retryCounter > Timing.HttpCallRetryCount())
{
Assert.Fail(exception.Message);
return null;
@ -108,7 +113,7 @@ namespace CodexDistTests.TestCore
}
catch (Exception exception)
{
if (retryCounter > 5)
if (retryCounter > Timing.HttpCallRetryCount())
{
Assert.Fail(exception.Message);
throw;

14
TestCore/Timing.cs
View File

@ -4,7 +4,12 @@
{
public static TimeSpan HttpCallTimeout()
{
return TimeSpan.FromMinutes(10);
return TimeSpan.FromSeconds(10);
}
public static int HttpCallRetryCount()
{
return 5;
}
public static void RetryDelay()
@ -12,9 +17,14 @@
Utils.Sleep(TimeSpan.FromSeconds(3));
}
public static void WaitForServiceDelay()
public static void WaitForK8sServiceDelay()
{
Utils.Sleep(TimeSpan.FromSeconds(1));
}
public static TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(5);
}
}
}