Merge branch 'feature/pluralize-codexnodes-in-pod'
This commit is contained in:
benbierens
commit
6fc3ea3fca
|
|
@ -1,61 +0,0 @@
|
|||
using k8s.Models;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public class ActiveNode
|
||||
{
|
||||
public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
|
||||
{
|
||||
Origin = origin;
|
||||
SelectorName = orderNumber.ToString().PadLeft(6, '0');
|
||||
Port = port;
|
||||
}
|
||||
|
||||
public OfflineCodexNode Origin { get; }
|
||||
public string SelectorName { get; }
|
||||
public int Port { get; }
|
||||
public V1Deployment? Deployment { get; set; }
|
||||
public V1Service? Service { get; set; }
|
||||
public List<string> ActivePodNames { get; } = new List<string>();
|
||||
|
||||
public V1ObjectMeta GetServiceMetadata()
|
||||
{
|
||||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-entrypoint-" + SelectorName,
|
||||
NamespaceProperty = K8sManager.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
public V1ObjectMeta GetDeploymentMetadata()
|
||||
{
|
||||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-node-" + SelectorName,
|
||||
NamespaceProperty = K8sManager.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
public Dictionary<string, string> GetSelector()
|
||||
{
|
||||
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
|
||||
}
|
||||
|
||||
public string GetContainerPortName()
|
||||
{
|
||||
//Caution, was: "codex-api-port" + SelectorName
|
||||
//but string length causes 'UnprocessableEntity' exception in k8s.
|
||||
return "api-" + SelectorName;
|
||||
}
|
||||
|
||||
public string GetContainerName()
|
||||
{
|
||||
return "codex-test-node";
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,10 +14,10 @@ namespace CodexDistTestCore
|
|||
return "b20483";
|
||||
}
|
||||
|
||||
public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNode node)
|
||||
public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment)
|
||||
{
|
||||
var formatter = new EnvFormatter();
|
||||
formatter.Create(node);
|
||||
formatter.Create(node, environment);
|
||||
return formatter.Result;
|
||||
}
|
||||
|
||||
|
|
@ -25,8 +25,13 @@ namespace CodexDistTestCore
|
|||
{
|
||||
public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
|
||||
|
||||
public void Create(OfflineCodexNode node)
|
||||
public void Create(OfflineCodexNodes node, CodexNodeContainer environment)
|
||||
{
|
||||
AddVar("API_PORT", environment.ApiPort.ToString());
|
||||
AddVar("DATA_DIR", environment.DataDir);
|
||||
AddVar("DISC_PORT", environment.DiscoveryPort.ToString());
|
||||
AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}");
|
||||
|
||||
if (node.BootstrapNode != null)
|
||||
{
|
||||
var debugInfo = node.BootstrapNode.GetDebugInfo();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public class CodexNodeContainer
|
||||
{
|
||||
public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir)
|
||||
{
|
||||
Name = name;
|
||||
ServicePort = servicePort;
|
||||
ServicePortName = servicePortName;
|
||||
ApiPort = apiPort;
|
||||
ContainerPortName = containerPortName;
|
||||
DiscoveryPort = discoveryPort;
|
||||
ListenPort = listenPort;
|
||||
DataDir = dataDir;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
public int ServicePort { get; }
|
||||
public string ServicePortName { get; }
|
||||
public int ApiPort { get; }
|
||||
public string ContainerPortName { get; }
|
||||
public int DiscoveryPort { get; }
|
||||
public int ListenPort { get; }
|
||||
public string DataDir { get; }
|
||||
}
|
||||
|
||||
public class CodexGroupNumberSource
|
||||
{
|
||||
private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0);
|
||||
private readonly NumberSource groupContainerNameSource = new NumberSource(1);
|
||||
private readonly NumberSource servicePortSource = new NumberSource(30001);
|
||||
|
||||
public int GetNextCodexNodeGroupNumber()
|
||||
{
|
||||
return codexNodeGroupNumberSource.GetNextNumber();
|
||||
}
|
||||
|
||||
public string GetNextServicePortName()
|
||||
{
|
||||
return $"node{groupContainerNameSource.GetNextNumber()}";
|
||||
}
|
||||
|
||||
public int GetNextServicePort()
|
||||
{
|
||||
return servicePortSource.GetNextNumber();
|
||||
}
|
||||
}
|
||||
|
||||
public class CodexNodeContainerFactory
|
||||
{
|
||||
private readonly NumberSource containerNameSource = new NumberSource(1);
|
||||
private readonly NumberSource codexPortSource = new NumberSource(8080);
|
||||
private readonly CodexGroupNumberSource groupContainerFactory;
|
||||
|
||||
public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory)
|
||||
{
|
||||
this.groupContainerFactory = groupContainerFactory;
|
||||
}
|
||||
|
||||
public CodexNodeContainer CreateNext()
|
||||
{
|
||||
var n = containerNameSource.GetNextNumber();
|
||||
return new CodexNodeContainer(
|
||||
name: $"codex-node{n}",
|
||||
servicePort: groupContainerFactory.GetNextServicePort(),
|
||||
servicePortName: groupContainerFactory.GetNextServicePortName(),
|
||||
apiPort: codexPortSource.GetNextNumber(),
|
||||
containerPortName: $"api-{n}",
|
||||
discoveryPort: codexPortSource.GetNextNumber(),
|
||||
listenPort: codexPortSource.GetNextNumber(),
|
||||
dataDir: $"datadir{n}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
using k8s.Models;
|
||||
using System.Collections;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
|
||||
{
|
||||
IOfflineCodexNodes BringOffline();
|
||||
IOnlineCodexNode this[int index] { get; }
|
||||
}
|
||||
|
||||
public class CodexNodeGroup : ICodexNodeGroup
|
||||
{
|
||||
private readonly IK8sManager k8SManager;
|
||||
|
||||
public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
|
||||
{
|
||||
OrderNumber = orderNumber;
|
||||
Origin = origin;
|
||||
this.k8SManager = k8SManager;
|
||||
Nodes = nodes;
|
||||
|
||||
foreach (var n in nodes) n.Group = this;
|
||||
}
|
||||
|
||||
public IOnlineCodexNode this[int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
return Nodes[index];
|
||||
}
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes BringOffline()
|
||||
{
|
||||
return k8SManager.BringOffline(this);
|
||||
}
|
||||
|
||||
public int OrderNumber { get; }
|
||||
public OfflineCodexNodes Origin { get; }
|
||||
public OnlineCodexNode[] Nodes { get; }
|
||||
public V1Deployment? Deployment { get; set; }
|
||||
public V1Service? Service { get; set; }
|
||||
public PodInfo? PodInfo { get; set; }
|
||||
|
||||
public CodexNodeContainer[] GetContainers()
|
||||
{
|
||||
return Nodes.Select(n => n.Container).ToArray();
|
||||
}
|
||||
|
||||
public IEnumerator<IOnlineCodexNode> GetEnumerator()
|
||||
{
|
||||
return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return Nodes.GetEnumerator();
|
||||
}
|
||||
|
||||
public V1ObjectMeta GetServiceMetadata()
|
||||
{
|
||||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-entrypoint-" + OrderNumber,
|
||||
NamespaceProperty = K8sOperations.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
public V1ObjectMeta GetDeploymentMetadata()
|
||||
{
|
||||
return new V1ObjectMeta
|
||||
{
|
||||
Name = "codex-test-node-" + OrderNumber,
|
||||
NamespaceProperty = K8sOperations.K8sNamespace
|
||||
};
|
||||
}
|
||||
|
||||
public Dictionary<string, string> GetSelector()
|
||||
{
|
||||
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
|
||||
}
|
||||
}
|
||||
|
||||
public class PodInfo
|
||||
{
|
||||
public PodInfo(string name, string ip)
|
||||
{
|
||||
Name = name;
|
||||
Ip = ip;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
public string Ip { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ namespace CodexDistTestCore
|
|||
[SetUpFixture]
|
||||
public abstract class DistTest
|
||||
{
|
||||
private TestLog log = null!;
|
||||
private FileManager fileManager = null!;
|
||||
private K8sManager k8sManager = null!;
|
||||
|
||||
|
|
@ -13,12 +14,23 @@ namespace CodexDistTestCore
|
|||
{
|
||||
// Previous test run may have been interrupted.
|
||||
// Begin by cleaning everything up.
|
||||
fileManager = new FileManager();
|
||||
k8sManager = new K8sManager(fileManager);
|
||||
log = new TestLog();
|
||||
fileManager = new FileManager(log);
|
||||
k8sManager = new K8sManager(log, fileManager);
|
||||
|
||||
try
|
||||
{
|
||||
k8sManager.DeleteAllResources();
|
||||
fileManager.DeleteAllTestFiles();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
GlobalTestFailure.HasFailed = true;
|
||||
log.Error($"Global setup cleanup failed with: {ex}");
|
||||
throw;
|
||||
}
|
||||
log.Log("Global setup cleanup successful");
|
||||
}
|
||||
|
||||
[SetUp]
|
||||
public void SetUpDistTest()
|
||||
|
|
@ -29,9 +41,9 @@ namespace CodexDistTestCore
|
|||
}
|
||||
else
|
||||
{
|
||||
TestLog.BeginTest();
|
||||
fileManager = new FileManager();
|
||||
k8sManager = new K8sManager(fileManager);
|
||||
log = new TestLog();
|
||||
fileManager = new FileManager(log);
|
||||
k8sManager = new K8sManager(log, fileManager);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -40,13 +52,13 @@ namespace CodexDistTestCore
|
|||
{
|
||||
try
|
||||
{
|
||||
TestLog.EndTest(k8sManager);
|
||||
log.EndTest(k8sManager);
|
||||
k8sManager.DeleteAllResources();
|
||||
fileManager.DeleteAllTestFiles();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Error("Cleanup failed: " + ex.Message);
|
||||
log.Error("Cleanup failed: " + ex.Message);
|
||||
GlobalTestFailure.HasFailed = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -56,9 +68,9 @@ namespace CodexDistTestCore
|
|||
return fileManager.GenerateTestFile(size);
|
||||
}
|
||||
|
||||
public IOfflineCodexNode SetupCodexNode()
|
||||
public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
|
||||
{
|
||||
return new OfflineCodexNode(k8sManager);
|
||||
return new OfflineCodexNodes(k8sManager, numberOfNodes);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,10 +15,12 @@ namespace CodexDistTestCore
|
|||
private const string Folder = "TestDataFiles";
|
||||
private readonly Random random = new Random();
|
||||
private readonly List<TestFile> activeFiles = new List<TestFile>();
|
||||
private readonly TestLog log;
|
||||
|
||||
public FileManager()
|
||||
public FileManager(TestLog log)
|
||||
{
|
||||
if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder);
|
||||
this.log = log;
|
||||
}
|
||||
|
||||
public TestFile CreateEmptyTestFile()
|
||||
|
|
@ -26,7 +28,6 @@ namespace CodexDistTestCore
|
|||
var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
|
||||
File.Create(result.Filename).Close();
|
||||
activeFiles.Add(result);
|
||||
TestLog.Log($"Created test file '{result.Filename}'.");
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -34,7 +35,7 @@ namespace CodexDistTestCore
|
|||
{
|
||||
var result = CreateEmptyTestFile();
|
||||
GenerateFileBytes(result, size);
|
||||
TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
|
||||
log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -73,6 +74,12 @@ namespace CodexDistTestCore
|
|||
|
||||
public string Filename { get; }
|
||||
|
||||
public long GetFileSize()
|
||||
{
|
||||
var info = new FileInfo(Filename);
|
||||
return info.Length;
|
||||
}
|
||||
|
||||
public void AssertIsEqual(TestFile? other)
|
||||
{
|
||||
if (other == null) Assert.Fail("TestFile is null.");
|
||||
|
|
|
|||
|
|
@ -20,18 +20,22 @@ namespace CodexDistTestCore
|
|||
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
|
||||
}
|
||||
|
||||
public T HttpGetJson<T>(string route)
|
||||
public string HttpGetString(string route)
|
||||
{
|
||||
return Retry(() =>
|
||||
{
|
||||
using var client = GetClient();
|
||||
var url = GetUrl() + route;
|
||||
var result = Utils.Wait(client.GetAsync(url));
|
||||
var json = Utils.Wait(result.Content.ReadAsStringAsync());
|
||||
return JsonConvert.DeserializeObject<T>(json)!;
|
||||
return Utils.Wait(result.Content.ReadAsStringAsync());
|
||||
});
|
||||
}
|
||||
|
||||
public T HttpGetJson<T>(string route)
|
||||
{
|
||||
return JsonConvert.DeserializeObject<T>(HttpGetString(route))!;
|
||||
}
|
||||
|
||||
public string HttpPostStream(string route, Stream stream)
|
||||
{
|
||||
return Retry(() =>
|
||||
|
|
@ -75,6 +79,7 @@ namespace CodexDistTestCore
|
|||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
Timing.HttpCallRetryDelay();
|
||||
retryCounter++;
|
||||
if (retryCounter > Timing.HttpCallRetryCount())
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,297 +1,86 @@
|
|||
using k8s;
|
||||
using k8s.Models;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IK8sManager
|
||||
{
|
||||
IOnlineCodexNode BringOnline(OfflineCodexNode node);
|
||||
IOfflineCodexNode BringOffline(IOnlineCodexNode node);
|
||||
ICodexNodeGroup BringOnline(OfflineCodexNodes node);
|
||||
IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
|
||||
}
|
||||
|
||||
public class K8sManager : IK8sManager
|
||||
{
|
||||
public const string K8sNamespace = "codex-test-namespace";
|
||||
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
|
||||
private readonly NumberSource numberSource = new NumberSource();
|
||||
private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
|
||||
private readonly List<string> knownActivePodNames = new List<string>();
|
||||
private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
|
||||
private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
|
||||
private readonly KnownK8sPods knownPods = new KnownK8sPods();
|
||||
private readonly TestLog log;
|
||||
private readonly IFileManager fileManager;
|
||||
|
||||
public K8sManager(IFileManager fileManager)
|
||||
public K8sManager(TestLog log, IFileManager fileManager)
|
||||
{
|
||||
this.log = log;
|
||||
this.fileManager = fileManager;
|
||||
}
|
||||
|
||||
public IOnlineCodexNode BringOnline(OfflineCodexNode node)
|
||||
public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
|
||||
{
|
||||
var client = CreateClient();
|
||||
var online = CreateOnlineCodexNodes(offline);
|
||||
|
||||
EnsureTestNamespace(client);
|
||||
K8s(k => k.BringOnline(online, offline));
|
||||
|
||||
var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber());
|
||||
var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port);
|
||||
activeNodes.Add(codexNode, activeNode);
|
||||
log.Log($"{online.Describe()} online.");
|
||||
|
||||
CreateDeployment(activeNode, client, node);
|
||||
CreateService(activeNode, client);
|
||||
|
||||
WaitUntilOnline(activeNode, client);
|
||||
TestLog.Log($"{activeNode.Describe()} online.");
|
||||
|
||||
return codexNode;
|
||||
return online;
|
||||
}
|
||||
|
||||
public IOfflineCodexNode BringOffline(IOnlineCodexNode node)
|
||||
public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
|
||||
{
|
||||
var client = CreateClient();
|
||||
var online = GetAndRemoveActiveNodeFor(node);
|
||||
|
||||
var activeNode = GetAndRemoveActiveNodeFor(node);
|
||||
K8s(k => k.BringOffline(online));
|
||||
|
||||
var deploymentName = activeNode.Deployment.Name();
|
||||
BringOffline(activeNode, client);
|
||||
WaitUntilOffline(deploymentName, client);
|
||||
TestLog.Log($"{activeNode.Describe()} offline.");
|
||||
log.Log($"{online.Describe()} offline.");
|
||||
|
||||
return activeNode.Origin;
|
||||
return online.Origin;
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
var client = CreateClient();
|
||||
|
||||
DeleteNamespace(client);
|
||||
|
||||
WaitUntilZeroPods(client);
|
||||
WaitUntilNamespaceDeleted(client);
|
||||
K8s(k => k.DeleteAllResources());
|
||||
}
|
||||
|
||||
public void FetchAllPodsLogs(Action<string, string, Stream> onLog)
|
||||
public void FetchAllPodsLogs(IPodLogsHandler logHandler)
|
||||
{
|
||||
var client = CreateClient();
|
||||
foreach (var node in activeNodes.Values)
|
||||
{
|
||||
var nodeDescription = node.Describe();
|
||||
foreach (var podName in node.ActivePodNames)
|
||||
{
|
||||
var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
|
||||
onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream);
|
||||
}
|
||||
}
|
||||
K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler));
|
||||
}
|
||||
|
||||
private void BringOffline(ActiveNode activeNode, Kubernetes client)
|
||||
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
|
||||
{
|
||||
DeleteDeployment(activeNode, client);
|
||||
DeleteService(activeNode, client);
|
||||
var containers = CreateContainers(offline.NumberOfNodes);
|
||||
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
|
||||
var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
|
||||
onlineCodexNodes.Add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
#region Waiting
|
||||
|
||||
private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
|
||||
private CodexNodeContainer[] CreateContainers(int number)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace);
|
||||
return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0;
|
||||
});
|
||||
|
||||
AssignActivePodNames(activeNode, client);
|
||||
var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
|
||||
var containers = new List<CodexNodeContainer>();
|
||||
for (var i = 0; i < number; i++) containers.Add(factory.CreateNext());
|
||||
return containers.ToArray();
|
||||
}
|
||||
|
||||
private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client)
|
||||
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
|
||||
{
|
||||
var pods = client.ListNamespacedPod(K8sNamespace);
|
||||
var podNames = pods.Items.Select(p => p.Name());
|
||||
foreach (var podName in podNames)
|
||||
{
|
||||
if (!knownActivePodNames.Contains(podName))
|
||||
{
|
||||
knownActivePodNames.Add(podName);
|
||||
activeNode.ActivePodNames.Add(podName);
|
||||
}
|
||||
}
|
||||
var n = (CodexNodeGroup)node;
|
||||
onlineCodexNodes.Remove(n);
|
||||
return n;
|
||||
}
|
||||
|
||||
private void WaitUntilOffline(string deploymentName, Kubernetes client)
|
||||
private void K8s(Action<K8sOperations> action)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
|
||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilZeroPods(Kubernetes client)
|
||||
{
|
||||
WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
|
||||
}
|
||||
|
||||
private void WaitUntilNamespaceDeleted(Kubernetes client)
|
||||
{
|
||||
WaitUntil(() => !IsTestNamespaceOnline(client));
|
||||
}
|
||||
|
||||
private void WaitUntil(Func<bool> predicate)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var state = predicate();
|
||||
while (!state)
|
||||
{
|
||||
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
|
||||
{
|
||||
Assert.Fail("K8s operation timed out.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Timing.WaitForK8sServiceDelay();
|
||||
state = predicate();
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Service management
|
||||
|
||||
private void CreateService(ActiveNode node, Kubernetes client)
|
||||
{
|
||||
var serviceSpec = new V1Service
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = node.GetServiceMetadata(),
|
||||
Spec = new V1ServiceSpec
|
||||
{
|
||||
Type = "NodePort",
|
||||
Selector = node.GetSelector(),
|
||||
Ports = new List<V1ServicePort>
|
||||
{
|
||||
new V1ServicePort
|
||||
{
|
||||
Protocol = "TCP",
|
||||
Port = 8080,
|
||||
TargetPort = node.GetContainerPortName(),
|
||||
NodePort = node.Port
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private void DeleteService(ActiveNode node, Kubernetes client)
|
||||
{
|
||||
if (node.Service == null) return;
|
||||
client.DeleteNamespacedService(node.Service.Name(), K8sNamespace);
|
||||
node.Service = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Deployment management
|
||||
|
||||
private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode)
|
||||
{
|
||||
var deploymentSpec = new V1Deployment
|
||||
{
|
||||
ApiVersion = "apps/v1",
|
||||
Metadata = node.GetDeploymentMetadata(),
|
||||
Spec = new V1DeploymentSpec
|
||||
{
|
||||
Replicas = 1,
|
||||
Selector = new V1LabelSelector
|
||||
{
|
||||
MatchLabels = node.GetSelector()
|
||||
},
|
||||
Template = new V1PodTemplateSpec
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Labels = node.GetSelector()
|
||||
},
|
||||
Spec = new V1PodSpec
|
||||
{
|
||||
Containers = new List<V1Container>
|
||||
{
|
||||
new V1Container
|
||||
{
|
||||
Name = node.GetContainerName(),
|
||||
Image = dockerImage.GetImageTag(),
|
||||
Ports = new List<V1ContainerPort>
|
||||
{
|
||||
new V1ContainerPort
|
||||
{
|
||||
ContainerPort = 8080,
|
||||
Name = node.GetContainerPortName()
|
||||
}
|
||||
},
|
||||
Env = dockerImage.CreateEnvironmentVariables(codexNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private void DeleteDeployment(ActiveNode node, Kubernetes client)
|
||||
{
|
||||
if (node.Deployment == null) return;
|
||||
client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace);
|
||||
node.Deployment = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Namespace management
|
||||
|
||||
private void EnsureTestNamespace(Kubernetes client)
|
||||
{
|
||||
if (IsTestNamespaceOnline(client)) return;
|
||||
|
||||
var namespaceSpec = new V1Namespace
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = K8sNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
||||
}
|
||||
};
|
||||
client.CreateNamespace(namespaceSpec);
|
||||
}
|
||||
|
||||
private void DeleteNamespace(Kubernetes client)
|
||||
{
|
||||
if (IsTestNamespaceOnline(client))
|
||||
{
|
||||
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private static Kubernetes CreateClient()
|
||||
{
|
||||
// todo: If the default KubeConfig file does not suffice, change it here:
|
||||
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
|
||||
return new Kubernetes(config);
|
||||
}
|
||||
|
||||
private static bool IsTestNamespaceOnline(Kubernetes client)
|
||||
{
|
||||
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
|
||||
}
|
||||
|
||||
private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
|
||||
{
|
||||
var n = (OnlineCodexNode)node;
|
||||
var activeNode = activeNodes[n];
|
||||
activeNodes.Remove(n);
|
||||
return activeNode;
|
||||
var k8s = new K8sOperations(knownPods);
|
||||
action(k8s);
|
||||
k8s.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,286 @@
|
|||
using k8s;
|
||||
using k8s.Models;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace CodexDistTestCore
|
||||
{
|
||||
public class K8sOperations
|
||||
{
|
||||
public const string K8sNamespace = "codex-test-namespace";
|
||||
|
||||
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
|
||||
private readonly Kubernetes client;
|
||||
private readonly KnownK8sPods knownPods;
|
||||
|
||||
public K8sOperations(KnownK8sPods knownPods)
|
||||
{
|
||||
this.knownPods = knownPods;
|
||||
|
||||
// todo: If the default KubeConfig file does not suffice, change it here:
|
||||
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
|
||||
client = new Kubernetes(config);
|
||||
}
|
||||
|
||||
public void Close()
|
||||
{
|
||||
client.Dispose();
|
||||
}
|
||||
|
||||
public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
|
||||
{
|
||||
EnsureTestNamespace();
|
||||
|
||||
CreateDeployment(online, offline);
|
||||
CreateService(online);
|
||||
|
||||
WaitUntilOnline(online);
|
||||
FetchPodInfo(online);
|
||||
}
|
||||
|
||||
public void BringOffline(CodexNodeGroup online)
|
||||
{
|
||||
var deploymentName = online.Deployment.Name();
|
||||
DeleteDeployment(online);
|
||||
DeleteService(online);
|
||||
WaitUntilOffline(deploymentName);
|
||||
}
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
DeleteNamespace();
|
||||
|
||||
WaitUntilZeroPods();
|
||||
WaitUntilNamespaceDeleted();
|
||||
}
|
||||
|
||||
public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
|
||||
{
|
||||
var logNumberSource = new NumberSource(0);
|
||||
foreach (var online in onlines)
|
||||
{
|
||||
foreach (var node in online)
|
||||
{
|
||||
WritePodLogs(online, node, logHandler, logNumberSource);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource)
|
||||
{
|
||||
var n = (OnlineCodexNode)node;
|
||||
var nodeDescription = $"{online.Describe()} contains {n.GetName()}";
|
||||
|
||||
var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name);
|
||||
logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream);
|
||||
}
|
||||
|
||||
private void FetchPodInfo(CodexNodeGroup online)
|
||||
{
|
||||
var pods = client.ListNamespacedPod(K8sNamespace).Items;
|
||||
|
||||
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
|
||||
Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
|
||||
|
||||
var newPod = newPods.Single();
|
||||
online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP);
|
||||
|
||||
Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure.");
|
||||
Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure.");
|
||||
|
||||
knownPods.Add(newPod.Name());
|
||||
}
|
||||
|
||||
#region Waiting
|
||||
|
||||
private void WaitUntilOnline(CodexNodeGroup online)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilOffline(string deploymentName)
|
||||
{
|
||||
WaitUntil(() =>
|
||||
{
|
||||
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
|
||||
return deployment == null || deployment.Status.AvailableReplicas == 0;
|
||||
});
|
||||
}
|
||||
|
||||
private void WaitUntilZeroPods()
|
||||
{
|
||||
WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
|
||||
}
|
||||
|
||||
private void WaitUntilNamespaceDeleted()
|
||||
{
|
||||
WaitUntil(() => !IsTestNamespaceOnline());
|
||||
}
|
||||
|
||||
private void WaitUntil(Func<bool> predicate)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
var state = predicate();
|
||||
while (!state)
|
||||
{
|
||||
if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
|
||||
{
|
||||
Assert.Fail("K8s operation timed out.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Timing.WaitForK8sServiceDelay();
|
||||
state = predicate();
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Service management
|
||||
|
||||
private void CreateService(CodexNodeGroup online)
|
||||
{
|
||||
var serviceSpec = new V1Service
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = online.GetServiceMetadata(),
|
||||
Spec = new V1ServiceSpec
|
||||
{
|
||||
Type = "NodePort",
|
||||
Selector = online.GetSelector(),
|
||||
Ports = CreateServicePorts(online)
|
||||
}
|
||||
};
|
||||
|
||||
online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1ServicePort> CreateServicePorts(CodexNodeGroup online)
|
||||
{
|
||||
var result = new List<V1ServicePort>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1ServicePort
|
||||
{
|
||||
Name = container.ServicePortName,
|
||||
Protocol = "TCP",
|
||||
Port = container.ApiPort,
|
||||
TargetPort = container.ContainerPortName,
|
||||
NodePort = container.ServicePort
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteService(CodexNodeGroup online)
|
||||
{
|
||||
if (online.Service == null) return;
|
||||
client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
|
||||
online.Service = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Deployment management
|
||||
|
||||
private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
|
||||
{
|
||||
var deploymentSpec = new V1Deployment
|
||||
{
|
||||
ApiVersion = "apps/v1",
|
||||
Metadata = online.GetDeploymentMetadata(),
|
||||
Spec = new V1DeploymentSpec
|
||||
{
|
||||
Replicas = 1,
|
||||
Selector = new V1LabelSelector
|
||||
{
|
||||
MatchLabels = online.GetSelector()
|
||||
},
|
||||
Template = new V1PodTemplateSpec
|
||||
{
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Labels = online.GetSelector()
|
||||
},
|
||||
Spec = new V1PodSpec
|
||||
{
|
||||
Containers = CreateDeploymentContainers(online, offline)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
|
||||
}
|
||||
|
||||
private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
|
||||
{
|
||||
var result = new List<V1Container>();
|
||||
var containers = online.GetContainers();
|
||||
foreach (var container in containers)
|
||||
{
|
||||
result.Add(new V1Container
|
||||
{
|
||||
Name = container.Name,
|
||||
Image = dockerImage.GetImageTag(),
|
||||
Ports = new List<V1ContainerPort>
|
||||
{
|
||||
new V1ContainerPort
|
||||
{
|
||||
ContainerPort = container.ApiPort,
|
||||
Name = container.ContainerPortName
|
||||
}
|
||||
},
|
||||
Env = dockerImage.CreateEnvironmentVariables(offline, container)
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void DeleteDeployment(CodexNodeGroup online)
|
||||
{
|
||||
if (online.Deployment == null) return;
|
||||
client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
|
||||
online.Deployment = null;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Namespace management
|
||||
|
||||
private void EnsureTestNamespace()
|
||||
{
|
||||
if (IsTestNamespaceOnline()) return;
|
||||
|
||||
var namespaceSpec = new V1Namespace
|
||||
{
|
||||
ApiVersion = "v1",
|
||||
Metadata = new V1ObjectMeta
|
||||
{
|
||||
Name = K8sNamespace,
|
||||
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
|
||||
}
|
||||
};
|
||||
client.CreateNamespace(namespaceSpec);
|
||||
}
|
||||
|
||||
private void DeleteNamespace()
|
||||
{
|
||||
if (IsTestNamespaceOnline())
|
||||
{
|
||||
client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private bool IsTestNamespaceOnline()
|
||||
{
|
||||
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public class KnownK8sPods
|
||||
{
|
||||
private readonly List<string> knownActivePodNames = new List<string>();
|
||||
|
||||
public bool Contains(string name)
|
||||
{
|
||||
return knownActivePodNames.Contains(name);
|
||||
}
|
||||
|
||||
public void Add(string name)
|
||||
{
|
||||
knownActivePodNames.Add(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,27 +2,18 @@
|
|||
{
|
||||
public class NumberSource
|
||||
{
|
||||
private int freePort;
|
||||
private int nodeOrderNumber;
|
||||
private int number;
|
||||
|
||||
public NumberSource()
|
||||
public NumberSource(int start)
|
||||
{
|
||||
freePort = 30001;
|
||||
nodeOrderNumber = 0;
|
||||
number = start;
|
||||
}
|
||||
|
||||
public int GetFreePort()
|
||||
public int GetNextNumber()
|
||||
{
|
||||
var port = freePort;
|
||||
freePort++;
|
||||
return port;
|
||||
}
|
||||
|
||||
public int GetNodeOrderNumber()
|
||||
{
|
||||
var number = nodeOrderNumber;
|
||||
nodeOrderNumber++;
|
||||
return number;
|
||||
var n = number;
|
||||
number++;
|
||||
return n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IOfflineCodexNode
|
||||
{
|
||||
IOfflineCodexNode WithLogLevel(CodexLogLevel level);
|
||||
IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node);
|
||||
IOfflineCodexNode WithStorageQuota(ByteSize storageQuota);
|
||||
IOnlineCodexNode BringOnline();
|
||||
}
|
||||
|
||||
public enum CodexLogLevel
|
||||
{
|
||||
Trace,
|
||||
Debug,
|
||||
Info,
|
||||
Warn,
|
||||
Error
|
||||
}
|
||||
|
||||
public class OfflineCodexNode : IOfflineCodexNode
|
||||
{
|
||||
private readonly IK8sManager k8SManager;
|
||||
|
||||
public CodexLogLevel? LogLevel { get; private set; }
|
||||
public IOnlineCodexNode? BootstrapNode { get; private set; }
|
||||
public ByteSize? StorageQuota { get; private set; }
|
||||
|
||||
public OfflineCodexNode(IK8sManager k8SManager)
|
||||
{
|
||||
this.k8SManager = k8SManager;
|
||||
}
|
||||
|
||||
public IOnlineCodexNode BringOnline()
|
||||
{
|
||||
return k8SManager.BringOnline(this);
|
||||
}
|
||||
|
||||
public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node)
|
||||
{
|
||||
BootstrapNode = node;
|
||||
return this;
|
||||
}
|
||||
|
||||
public IOfflineCodexNode WithLogLevel(CodexLogLevel level)
|
||||
{
|
||||
LogLevel = level;
|
||||
return this;
|
||||
}
|
||||
|
||||
public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota)
|
||||
{
|
||||
StorageQuota = storageQuota;
|
||||
return this;
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
var result = "";
|
||||
if (LogLevel != null) result += $"LogLevel={LogLevel},";
|
||||
if (BootstrapNode != null) result += "BootstrapNode=set,";
|
||||
if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},";
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IOfflineCodexNodes
|
||||
{
|
||||
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
|
||||
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
|
||||
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
|
||||
ICodexNodeGroup BringOnline();
|
||||
}
|
||||
|
||||
public enum CodexLogLevel
|
||||
{
|
||||
Trace,
|
||||
Debug,
|
||||
Info,
|
||||
Warn,
|
||||
Error
|
||||
}
|
||||
|
||||
public class OfflineCodexNodes : IOfflineCodexNodes
|
||||
{
|
||||
private readonly IK8sManager k8SManager;
|
||||
|
||||
public int NumberOfNodes { get; }
|
||||
public CodexLogLevel? LogLevel { get; private set; }
|
||||
public IOnlineCodexNode? BootstrapNode { get; private set; }
|
||||
public ByteSize? StorageQuota { get; private set; }
|
||||
|
||||
public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
|
||||
{
|
||||
this.k8SManager = k8SManager;
|
||||
NumberOfNodes = numberOfNodes;
|
||||
}
|
||||
|
||||
public ICodexNodeGroup BringOnline()
|
||||
{
|
||||
return k8SManager.BringOnline(this);
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
|
||||
{
|
||||
BootstrapNode = node;
|
||||
return this;
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes WithLogLevel(CodexLogLevel level)
|
||||
{
|
||||
LogLevel = level;
|
||||
return this;
|
||||
}
|
||||
|
||||
public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota)
|
||||
{
|
||||
StorageQuota = storageQuota;
|
||||
return this;
|
||||
}
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
var args = string.Join(',', DescribeArgs());
|
||||
return $"{NumberOfNodes} CodexNodes with [{args}]";
|
||||
}
|
||||
|
||||
private IEnumerable<string> DescribeArgs()
|
||||
{
|
||||
if (LogLevel != null) yield return ($"LogLevel={LogLevel}");
|
||||
if (BootstrapNode != null) yield return ("BootstrapNode=set");
|
||||
if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,55 +7,106 @@ namespace CodexDistTestCore
|
|||
CodexDebugResponse GetDebugInfo();
|
||||
ContentId UploadFile(TestFile file);
|
||||
TestFile? DownloadContent(ContentId contentId);
|
||||
IOfflineCodexNode BringOffline();
|
||||
void ConnectToPeer(IOnlineCodexNode node);
|
||||
}
|
||||
|
||||
public class OnlineCodexNode : IOnlineCodexNode
|
||||
{
|
||||
private readonly IK8sManager k8SManager;
|
||||
private readonly IFileManager fileManager;
|
||||
private readonly int port;
|
||||
private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
|
||||
private const string UploadFailedMessage = "Unable to store block";
|
||||
|
||||
public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port)
|
||||
private readonly TestLog log;
|
||||
private readonly IFileManager fileManager;
|
||||
|
||||
public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container)
|
||||
{
|
||||
this.k8SManager = k8SManager;
|
||||
this.log = log;
|
||||
this.fileManager = fileManager;
|
||||
this.port = port;
|
||||
Container = container;
|
||||
}
|
||||
|
||||
public IOfflineCodexNode BringOffline()
|
||||
public CodexNodeContainer Container { get; }
|
||||
public CodexNodeGroup Group { get; internal set; } = null!;
|
||||
|
||||
public string GetName()
|
||||
{
|
||||
return k8SManager.BringOffline(this);
|
||||
return $"<{Container.Name}>";
|
||||
}
|
||||
|
||||
public CodexDebugResponse GetDebugInfo()
|
||||
{
|
||||
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
|
||||
var response = Http().HttpGetJson<CodexDebugResponse>("debug/info");
|
||||
Log($"Got DebugInfo with id: '{response.id}'.");
|
||||
return response;
|
||||
}
|
||||
|
||||
public ContentId UploadFile(TestFile file)
|
||||
{
|
||||
Log($"Uploading file of size {file.GetFileSize()}...");
|
||||
using var fileStream = File.OpenRead(file.Filename);
|
||||
var response = Http().HttpPostStream("upload", fileStream);
|
||||
if (response.StartsWith("Unable to store block"))
|
||||
if (response.StartsWith(UploadFailedMessage))
|
||||
{
|
||||
Assert.Fail("Node failed to store block.");
|
||||
}
|
||||
Log($"Uploaded file. Received contentId: '{response}'.");
|
||||
return new ContentId(response);
|
||||
}
|
||||
|
||||
public TestFile? DownloadContent(ContentId contentId)
|
||||
{
|
||||
Log($"Downloading for contentId: '{contentId.Id}'...");
|
||||
var file = fileManager.CreateEmptyTestFile();
|
||||
using var fileStream = File.OpenWrite(file.Filename);
|
||||
using var downloadStream = Http().HttpGetStream("download/" + contentId.Id);
|
||||
downloadStream.CopyTo(fileStream);
|
||||
DownloadToFile(contentId.Id, file);
|
||||
Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
|
||||
return file;
|
||||
}
|
||||
|
||||
public void ConnectToPeer(IOnlineCodexNode node)
|
||||
{
|
||||
var peer = (OnlineCodexNode)node;
|
||||
|
||||
Log($"Connecting to peer {peer.GetName()}...");
|
||||
var peerInfo = node.GetDebugInfo();
|
||||
var peerId = peerInfo.id;
|
||||
var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo);
|
||||
|
||||
var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
|
||||
|
||||
Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
|
||||
Log($"Successfully connected to peer {peer.GetName()}.");
|
||||
}
|
||||
|
||||
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
|
||||
{
|
||||
var multiAddress = peerInfo.addrs.First();
|
||||
// Todo: Is there a case where First address in list is not the way?
|
||||
|
||||
if (Group == peer.Group)
|
||||
{
|
||||
return multiAddress;
|
||||
}
|
||||
|
||||
// The peer we want to connect is in a different pod.
|
||||
// We must replace the default IP with the pod IP in the multiAddress.
|
||||
return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip);
|
||||
}
|
||||
|
||||
private void DownloadToFile(string contentId, TestFile file)
|
||||
{
|
||||
using var fileStream = File.OpenWrite(file.Filename);
|
||||
using var downloadStream = Http().HttpGetStream("download/" + contentId);
|
||||
downloadStream.CopyTo(fileStream);
|
||||
}
|
||||
|
||||
private Http Http()
|
||||
{
|
||||
return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1");
|
||||
return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log($"{GetName()}: {msg}");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
namespace CodexDistTestCore
|
||||
{
|
||||
public interface IPodLogsHandler
|
||||
{
|
||||
void Log(int id, string podDescription, Stream log);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,75 +5,94 @@ namespace CodexDistTestCore
|
|||
public class TestLog
|
||||
{
|
||||
public const string LogRoot = "D:/CodexTestLogs";
|
||||
private readonly LogFile file;
|
||||
|
||||
private static LogFile? file = null;
|
||||
|
||||
public static void Log(string message)
|
||||
public TestLog()
|
||||
{
|
||||
file!.Write(message);
|
||||
}
|
||||
|
||||
public static void Error(string message)
|
||||
{
|
||||
Log($"[ERROR] {message}");
|
||||
}
|
||||
|
||||
public static void BeginTest()
|
||||
{
|
||||
if (file != null) throw new InvalidOperationException("Test is already started!");
|
||||
|
||||
var name = GetTestName();
|
||||
file = new LogFile(name);
|
||||
|
||||
Log($"Begin: {name}");
|
||||
}
|
||||
|
||||
public static void EndTest(K8sManager k8sManager)
|
||||
public void Log(string message)
|
||||
{
|
||||
if (file == null) throw new InvalidOperationException("No test is started!");
|
||||
file.Write(message);
|
||||
}
|
||||
|
||||
public void Error(string message)
|
||||
{
|
||||
Log($"[ERROR] {message}");
|
||||
}
|
||||
|
||||
public void EndTest(K8sManager k8sManager)
|
||||
{
|
||||
var result = TestContext.CurrentContext.Result;
|
||||
|
||||
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
|
||||
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
||||
|
||||
if (!string.IsNullOrEmpty(result.Message))
|
||||
{
|
||||
IncludeFullPodLogging(k8sManager);
|
||||
Log(result.Message);
|
||||
}
|
||||
|
||||
file = null;
|
||||
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
||||
{
|
||||
Log($"{result.StackTrace}");
|
||||
|
||||
var logWriter = new PodLogWriter(file);
|
||||
logWriter.IncludeFullPodLogging(k8sManager);
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetTestName()
|
||||
{
|
||||
var test = TestContext.CurrentContext.Test;
|
||||
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
|
||||
return $"{className}.{test.MethodName}";
|
||||
var args = FormatArguments(test);
|
||||
return $"{className}.{test.MethodName}{args}";
|
||||
}
|
||||
|
||||
private static void LogRaw(string message, string filename)
|
||||
private static string FormatArguments(TestContext.TestAdapter test)
|
||||
{
|
||||
file!.WriteRaw(message, filename);
|
||||
if (test.Arguments == null || !test.Arguments.Any()) return "";
|
||||
return $"[{string.Join(',', test.Arguments)}]";
|
||||
}
|
||||
}
|
||||
|
||||
private static void IncludeFullPodLogging(K8sManager k8sManager)
|
||||
public class PodLogWriter : IPodLogsHandler
|
||||
{
|
||||
Log("Full pod logging:");
|
||||
k8sManager.FetchAllPodsLogs(WritePodLog);
|
||||
private readonly LogFile file;
|
||||
|
||||
public PodLogWriter(LogFile file)
|
||||
{
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
private static void WritePodLog(string id, string nodeDescription, Stream stream)
|
||||
public void IncludeFullPodLogging(K8sManager k8sManager)
|
||||
{
|
||||
Log($"{nodeDescription} -->> {id}");
|
||||
LogRaw(nodeDescription, id);
|
||||
var reader = new StreamReader(stream);
|
||||
file.Write("Full pod logging:");
|
||||
k8sManager.FetchAllPodsLogs(this);
|
||||
}
|
||||
|
||||
public void Log(int id, string podDescription, Stream log)
|
||||
{
|
||||
var logFile = id.ToString().PadLeft(6, '0');
|
||||
file.Write($"{podDescription} -->> {logFile}");
|
||||
LogRaw(podDescription, logFile);
|
||||
var reader = new StreamReader(log);
|
||||
var line = reader.ReadLine();
|
||||
while (line != null)
|
||||
{
|
||||
LogRaw(line, id);
|
||||
LogRaw(line, logFile);
|
||||
line = reader.ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
private void LogRaw(string message, string filename)
|
||||
{
|
||||
file!.WriteRaw(message, filename);
|
||||
}
|
||||
}
|
||||
|
||||
public class LogFile
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ namespace CodexDistTestCore
|
|||
return GetTimes().HttpCallRetryCount();
|
||||
}
|
||||
|
||||
public static void RetryDelay()
|
||||
public static void HttpCallRetryDelay()
|
||||
{
|
||||
Utils.Sleep(GetTimes().RetryDelay());
|
||||
Utils.Sleep(GetTimes().HttpCallRetryDelay());
|
||||
}
|
||||
|
||||
public static void WaitForK8sServiceDelay()
|
||||
|
|
@ -52,7 +52,7 @@ namespace CodexDistTestCore
|
|||
{
|
||||
TimeSpan HttpCallTimeout();
|
||||
int HttpCallRetryCount();
|
||||
TimeSpan RetryDelay();
|
||||
TimeSpan HttpCallRetryDelay();
|
||||
TimeSpan WaitForK8sServiceDelay();
|
||||
TimeSpan K8sOperationTimeout();
|
||||
}
|
||||
|
|
@ -69,7 +69,7 @@ namespace CodexDistTestCore
|
|||
return 5;
|
||||
}
|
||||
|
||||
public TimeSpan RetryDelay()
|
||||
public TimeSpan HttpCallRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromSeconds(3);
|
||||
}
|
||||
|
|
@ -97,7 +97,7 @@ namespace CodexDistTestCore
|
|||
return 2;
|
||||
}
|
||||
|
||||
public TimeSpan RetryDelay()
|
||||
public TimeSpan HttpCallRetryDelay()
|
||||
{
|
||||
return TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,17 +4,17 @@ using NUnit.Framework;
|
|||
namespace LongTests.BasicTests
|
||||
{
|
||||
[TestFixture]
|
||||
public class SimpleTests : DistTest
|
||||
public class LargeFileTests : DistTest
|
||||
{
|
||||
[Test, UseLongTimeouts]
|
||||
public void OneClientLargeFileTest()
|
||||
{
|
||||
var primary = SetupCodexNode()
|
||||
var primary = SetupCodexNodes(1)
|
||||
.WithLogLevel(CodexLogLevel.Warn)
|
||||
.WithStorageQuota(10.GB())
|
||||
.BringOnline();
|
||||
.WithStorageQuota(20.GB())
|
||||
.BringOnline()[0];
|
||||
|
||||
var testFile = GenerateTestFile(1.GB());
|
||||
var testFile = GenerateTestFile(10.GB());
|
||||
|
||||
var contentId = primary.UploadFile(testFile);
|
||||
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
using CodexDistTestCore;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace LongTests.BasicTests
|
||||
{
|
||||
public class TestInfraTests : DistTest
|
||||
{
|
||||
[Test, UseLongTimeouts]
|
||||
public void TestInfraShouldHave1000AddressSpacesPerPod()
|
||||
{
|
||||
var group = SetupCodexNodes(1000).BringOnline();
|
||||
|
||||
var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();
|
||||
|
||||
Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()),
|
||||
"Not all created nodes provided a unique id.");
|
||||
}
|
||||
|
||||
[Test, UseLongTimeouts]
|
||||
public void TestInfraSupportsManyConcurrentPods()
|
||||
{
|
||||
for (var i = 0; i < 20; i++)
|
||||
{
|
||||
var n = SetupCodexNodes(1).BringOnline()[0];
|
||||
|
||||
Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id));
|
||||
}
|
||||
}
|
||||
|
||||
[Test, UseLongTimeouts]
|
||||
public void DownloadConsistencyTest()
|
||||
{
|
||||
var primary = SetupCodexNodes(1)
|
||||
.WithLogLevel(CodexLogLevel.Trace)
|
||||
.WithStorageQuota(2.MB())
|
||||
.BringOnline()[0];
|
||||
|
||||
var testFile = GenerateTestFile(1.MB());
|
||||
|
||||
var contentId = primary.UploadFile(testFile);
|
||||
|
||||
var files = new List<TestFile?>();
|
||||
for (var i = 0; i < 100; i++)
|
||||
{
|
||||
files.Add(primary.DownloadContent(contentId));
|
||||
}
|
||||
|
||||
Assert.That(files.All(f => f != null));
|
||||
Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize()));
|
||||
foreach (var file in files) file!.AssertIsEqual(testFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,7 +11,7 @@ namespace Tests.BasicTests
|
|||
{
|
||||
var dockerImage = new CodexDockerImage();
|
||||
|
||||
var node = SetupCodexNode().BringOnline();
|
||||
var node = SetupCodexNodes(1).BringOnline()[0];
|
||||
|
||||
var debugInfo = node.GetDebugInfo();
|
||||
|
||||
|
|
@ -22,10 +22,10 @@ namespace Tests.BasicTests
|
|||
[Test]
|
||||
public void OneClientTest()
|
||||
{
|
||||
var primary = SetupCodexNode()
|
||||
var primary = SetupCodexNodes(1)
|
||||
.WithLogLevel(CodexLogLevel.Trace)
|
||||
.WithStorageQuota(2.MB())
|
||||
.BringOnline();
|
||||
.BringOnline()[0];
|
||||
|
||||
var testFile = GenerateTestFile(1.MB());
|
||||
|
||||
|
|
@ -36,26 +36,45 @@ namespace Tests.BasicTests
|
|||
testFile.AssertIsEqual(downloadedFile);
|
||||
}
|
||||
|
||||
//[Test]
|
||||
//public void TwoClientTest()
|
||||
//{
|
||||
// var primary = SetupCodexNode()
|
||||
// .WithLogLevel(CodexLogLevel.Trace)
|
||||
// .WithStorageQuota(1024 * 1024 * 2)
|
||||
// .BringOnline();
|
||||
[Test]
|
||||
public void TwoClientOnePodTest()
|
||||
{
|
||||
var group = SetupCodexNodes(2)
|
||||
.WithLogLevel(CodexLogLevel.Trace)
|
||||
.WithStorageQuota(2.MB())
|
||||
.BringOnline();
|
||||
|
||||
// var secondary = SetupCodexNode()
|
||||
// .WithLogLevel(CodexLogLevel.Trace)
|
||||
// .WithBootstrapNode(primary)
|
||||
// .BringOnline();
|
||||
var primary = group[0];
|
||||
var secondary = group[1];
|
||||
|
||||
// var testFile = GenerateTestFile(1024 * 1024);
|
||||
PerformTwoClientTest(primary, secondary);
|
||||
}
|
||||
|
||||
// var contentId = primary.UploadFile(testFile);
|
||||
[Test]
|
||||
public void TwoClientTwoPodTest()
|
||||
{
|
||||
var primary = SetupCodexNodes(1)
|
||||
.WithStorageQuota(2.MB())
|
||||
.BringOnline()[0];
|
||||
|
||||
// var downloadedFile = secondary.DownloadContent(contentId);
|
||||
var secondary = SetupCodexNodes(1)
|
||||
.WithStorageQuota(2.MB())
|
||||
.BringOnline()[0];
|
||||
|
||||
// testFile.AssertIsEqual(downloadedFile);
|
||||
//}
|
||||
PerformTwoClientTest(primary, secondary);
|
||||
}
|
||||
|
||||
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
|
||||
{
|
||||
primary.ConnectToPeer(secondary);
|
||||
|
||||
var testFile = GenerateTestFile(1.MB());
|
||||
|
||||
var contentId = primary.UploadFile(testFile);
|
||||
|
||||
var downloadedFile = secondary.DownloadContent(contentId);
|
||||
|
||||
testFile.AssertIsEqual(downloadedFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,28 +1,3 @@
|
|||
# apiVersion: v1
|
||||
# kind: Pod
|
||||
# metadata:
|
||||
# name: codex-pod
|
||||
# spec:
|
||||
# containers:
|
||||
# - name: codex-node-container
|
||||
# image: thatbenbierens/nim-codex:sha-c9a62de
|
||||
# ports:
|
||||
# - containerPort: 8080
|
||||
|
||||
# kind: PersistentVolumeClaim
|
||||
# apiVersion: v1
|
||||
# metadata:
|
||||
# name: codex-volume
|
||||
# spec:
|
||||
# accessModes:
|
||||
# - ReadWriteOnce
|
||||
# resources:
|
||||
# requests:
|
||||
# storage: 1Gi
|
||||
# storageClassName: rbd
|
||||
|
||||
# ---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
|
|
@ -48,21 +23,35 @@ spec:
|
|||
codex-node: dist-test
|
||||
spec:
|
||||
containers:
|
||||
- name: codex-node
|
||||
image: thatbenbierens/nim-codex:sha-c9a62de
|
||||
- name: codex-node1
|
||||
image: thatbenbierens/nim-codex:sha-b204837
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: codex-api-port
|
||||
name: api-1
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: WARN
|
||||
# volumeMounts:
|
||||
# - mountPath: /datadir
|
||||
# name: codex-data
|
||||
# volumes:
|
||||
# - name: codex-data
|
||||
# persistentVolumeClaim:
|
||||
# claimName: codex-volume
|
||||
- name: API_PORT
|
||||
value: "8080"
|
||||
- name: DATA_DIR
|
||||
value: datadir1
|
||||
- name: DISC_PORT
|
||||
value: "8081"
|
||||
- name: LISTEN_ADDRS
|
||||
value: "/ip4/0.0.0.0/tcp/8082"
|
||||
- name: codex-node2
|
||||
image: thatbenbierens/nim-codex:sha-b204837
|
||||
ports:
|
||||
- containerPort: 8083
|
||||
name: api-2
|
||||
env:
|
||||
- name: API_PORT
|
||||
value: "8083"
|
||||
- name: DATA_DIR
|
||||
value: datadir2
|
||||
- name: DISC_PORT
|
||||
value: "8084"
|
||||
- name: LISTEN_ADDRS
|
||||
value: "/ip4/0.0.0.0/tcp/8085"
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
|
|
@ -75,7 +64,14 @@ spec:
|
|||
selector:
|
||||
codex-node: dist-test
|
||||
ports:
|
||||
- protocol: TCP
|
||||
- name: "node1"
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: codex-api-port
|
||||
targetPort: api-1
|
||||
nodePort: 30001
|
||||
- name: "node2"
|
||||
protocol: TCP
|
||||
port: 8083
|
||||
targetPort: api-2
|
||||
nodePort: 30002
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue