status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

this works

This commit is contained in:
benbierens 2023-03-21 15:17:48 +01:00
parent ca0bc9570b
commit 82e29d02c9
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
11 changed files with 207 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
CodexDistTestCore/ActiveNode.cs → CodexDistTestCore/ActiveDeployment.cs
View File

@ -2,18 +2,18 @@
namespace CodexDistTestCore
{
public class ActiveNode
public class ActiveDeployment
{
public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers)
{
Origin = origin;
Containers = containers;
SelectorName = orderNumber.ToString().PadLeft(6, '0');
Port = port;
}
public OfflineCodexNode Origin { get; }
public OfflineCodexNodes Origin { get; }
public CodexNodeContainer[] Containers { get; }
public string SelectorName { get; }
public int Port { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public List<string> ActivePodNames { get; } = new List<string>();
@ -41,21 +41,9 @@ namespace CodexDistTestCore
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
}
public string GetContainerPortName()
{
//Caution, was: "codex-api-port" + SelectorName
//but string length causes 'UnprocessableEntity' exception in k8s.
return "api-" + SelectorName;
}
public string GetContainerName()
{
return "codex-test-node";
}
public string Describe()
{
return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
return $"CodexNode{SelectorName}-{Origin.Describe()}";
}
}
}

11
CodexDistTestCore/CodexDockerImage.cs
View File

@ -14,10 +14,10 @@ namespace CodexDistTestCore
return "b20483";
}
public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNode node)
public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment)
{
var formatter = new EnvFormatter();
formatter.Create(node);
formatter.Create(node, environment);
return formatter.Result;
}
@ -25,8 +25,13 @@ namespace CodexDistTestCore
{
public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
public void Create(OfflineCodexNode node)
public void Create(OfflineCodexNodes node, CodexNodeContainer environment)
{
AddVar("API_PORT", environment.ApiPort.ToString());
AddVar("DATA_DIR", environment.DataDir);
AddVar("DISC_PORT", environment.DiscoveryPort.ToString());
AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}");
if (node.BootstrapNode != null)
{
var debugInfo = node.BootstrapNode.GetDebugInfo();

45
CodexDistTestCore/CodexNodeContainer.cs Normal file
View File

@ -0,0 +1,45 @@
namespace CodexDistTestCore
{
public class CodexNodeContainer
{
public CodexNodeContainer(string name, int servicePort, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir)
{
Name = name;
ServicePort = servicePort;
ApiPort = apiPort;
ContainerPortName = containerPortName;
DiscoveryPort = discoveryPort;
ListenPort = listenPort;
DataDir = dataDir;
}
public string Name { get; }
public int ServicePort { get; }
public int ApiPort { get; }
public string ContainerPortName { get; }
public int DiscoveryPort { get; }
public int ListenPort { get; }
public string DataDir { get; }
}
public class CodexNodeContainerFactory
{
private readonly NumberSource containerNameSource = new NumberSource(1);
private readonly NumberSource servicePortSource = new NumberSource(30001);
private readonly NumberSource codexPortSource = new NumberSource(8080);
public CodexNodeContainer CreateNext()
{
var n = containerNameSource.GetNextNumber();
return new CodexNodeContainer(
name: $"codex-node{n}",
servicePort: servicePortSource.GetNextNumber(),
apiPort: codexPortSource.GetNextNumber(),
containerPortName: $"api-{n}",
discoveryPort: codexPortSource.GetNextNumber(),
listenPort: codexPortSource.GetNextNumber(),
dataDir: $"datadir{n}"
);
}
}
}

4
CodexDistTestCore/DistTest.cs
View File

@ -56,9 +56,9 @@ namespace CodexDistTestCore
return fileManager.GenerateTestFile(size);
}
public IOfflineCodexNode SetupCodexNode()
public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
{
return new OfflineCodexNode(k8sManager);
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
}

140
CodexDistTestCore/K8sManager.cs
View File

@ -6,16 +6,16 @@ namespace CodexDistTestCore
{
public interface IK8sManager
{
IOnlineCodexNode BringOnline(OfflineCodexNode node);
IOfflineCodexNode BringOffline(IOnlineCodexNode node);
IOnlineCodexNodes BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(IOnlineCodexNodes node);
}
public class K8sManager : IK8sManager
{
public const string K8sNamespace = "codex-test-namespace";
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
private readonly NumberSource numberSource = new NumberSource();
private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0);
private readonly Dictionary<OnlineCodexNodes, ActiveDeployment> activeDeployments = new Dictionary<OnlineCodexNodes, ActiveDeployment>();
private readonly List<string> knownActivePodNames = new List<string>();
private readonly IFileManager fileManager;
@ -24,26 +24,38 @@ namespace CodexDistTestCore
this.fileManager = fileManager;
}
public IOnlineCodexNode BringOnline(OfflineCodexNode node)
public IOnlineCodexNodes BringOnline(OfflineCodexNodes node)
{
var client = CreateClient();
EnsureTestNamespace(client);
var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber());
var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port);
activeNodes.Add(codexNode, activeNode);
var factory = new CodexNodeContainerFactory();
var list = new List<OnlineCodexNode>();
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < node.NumberOfNodes; i++)
{
var container = factory.CreateNext();
containers.Add(container);
CreateDeployment(activeNode, client, node);
CreateService(activeNode, client);
var codexNode = new OnlineCodexNode(fileManager, container);
list.Add(codexNode);
}
WaitUntilOnline(activeNode, client);
TestLog.Log($"{activeNode.Describe()} online.");
var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray());
return codexNode;
var result = new OnlineCodexNodes(this, list.ToArray());
activeDeployments.Add(result, activeDeployment);
CreateDeployment(activeDeployment, client, node);
CreateService(activeDeployment, client);
WaitUntilOnline(activeDeployment, client);
TestLog.Log($"{node.NumberOfNodes} Codex nodes online.");
return result;
}
public IOfflineCodexNode BringOffline(IOnlineCodexNode node)
public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
{
var client = CreateClient();
@ -70,7 +82,7 @@ namespace CodexDistTestCore
public void FetchAllPodsLogs(Action<string, string, Stream> onLog)
{
var client = CreateClient();
foreach (var node in activeNodes.Values)
foreach (var node in activeDeployments.Values)
{
var nodeDescription = node.Describe();
foreach (var podName in node.ActivePodNames)
@ -81,7 +93,7 @@ namespace CodexDistTestCore
}
}
private void BringOffline(ActiveNode activeNode, Kubernetes client)
private void BringOffline(ActiveDeployment activeNode, Kubernetes client)
{
DeleteDeployment(activeNode, client);
DeleteService(activeNode, client);
@ -89,7 +101,7 @@ namespace CodexDistTestCore
#region Waiting
private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client)
{
WaitUntil(() =>
{
@ -100,7 +112,7 @@ namespace CodexDistTestCore
AssignActivePodNames(activeNode, client);
}
private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client)
private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client)
{
var pods = client.ListNamespacedPod(K8sNamespace);
var podNames = pods.Items.Select(p => p.Name());
@ -154,33 +166,40 @@ namespace CodexDistTestCore
#region Service management
private void CreateService(ActiveNode node, Kubernetes client)
private void CreateService(ActiveDeployment activeDeployment, Kubernetes client)
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = node.GetServiceMetadata(),
Metadata = activeDeployment.GetServiceMetadata(),
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = node.GetSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Protocol = "TCP",
Port = 8080,
TargetPort = node.GetContainerPortName(),
NodePort = node.Port
}
}
Selector = activeDeployment.GetSelector(),
Ports = CreateServicePorts(activeDeployment)
}
};
node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
}
private void DeleteService(ActiveNode node, Kubernetes client)
private List<V1ServicePort> CreateServicePorts(ActiveDeployment activeDeployment)
{
var result = new List<V1ServicePort>();
foreach (var container in activeDeployment.Containers)
{
result.Add(new V1ServicePort
{
Protocol = "TCP",
Port = 8080,
TargetPort = container.ContainerPortName,
NodePort = container.ServicePort
});
}
return result;
}
private void DeleteService(ActiveDeployment node, Kubernetes client)
{
if (node.Service == null) return;
client.DeleteNamespacedService(node.Service.Name(), K8sNamespace);
@ -191,7 +210,7 @@ namespace CodexDistTestCore
#region Deployment management
private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode)
private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode)
{
var deploymentSpec = new V1Deployment
{
@ -212,23 +231,7 @@ namespace CodexDistTestCore
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = node.GetContainerName(),
Image = dockerImage.GetImageTag(),
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 8080,
Name = node.GetContainerPortName()
}
},
Env = dockerImage.CreateEnvironmentVariables(codexNode)
}
}
Containers = CreateDeploymentContainers(node, codexNode)
}
}
}
@ -237,7 +240,30 @@ namespace CodexDistTestCore
node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
}
private void DeleteDeployment(ActiveNode node, Kubernetes client)
private List<V1Container> CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode)
{
var result = new List<V1Container>();
foreach (var container in node.Containers)
{
result.Add(new V1Container
{
Name = container.Name,
Image = dockerImage.GetImageTag(),
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = container.ApiPort,
Name = container.ContainerPortName
}
},
Env = dockerImage.CreateEnvironmentVariables(codexNode, container)
});
}
return result;
}
private void DeleteDeployment(ActiveDeployment node, Kubernetes client)
{
if (node.Deployment == null) return;
client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace);
@ -286,11 +312,11 @@ namespace CodexDistTestCore
return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
}
private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
{
var n = (OnlineCodexNode)node;
var activeNode = activeNodes[n];
activeNodes.Remove(n);
var n = (OnlineCodexNodes)node;
var activeNode = activeDeployments[n];
activeDeployments.Remove(n);
return activeNode;
}
}

23
CodexDistTestCore/NumberSource.cs
View File

@ -2,27 +2,18 @@
{
public class NumberSource
{
private int freePort;
private int nodeOrderNumber;
private int number;
public NumberSource()
public NumberSource(int start)
{
freePort = 30001;
nodeOrderNumber = 0;
number = start;
}
public int GetFreePort()
public int GetNextNumber()
{
var port = freePort;
freePort++;
return port;
}
public int GetNodeOrderNumber()
{
var number = nodeOrderNumber;
nodeOrderNumber++;
return number;
var n = number;
number++;
return n;
}
}
}

24
CodexDistTestCore/OfflineCodexNode.cs → CodexDistTestCore/OfflineCodexNodes.cs
View File

@ -1,11 +1,11 @@
namespace CodexDistTestCore
{
public interface IOfflineCodexNode
public interface IOfflineCodexNodes
{
IOfflineCodexNode WithLogLevel(CodexLogLevel level);
IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNode WithStorageQuota(ByteSize storageQuota);
IOnlineCodexNode BringOnline();
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
IOnlineCodexNodes BringOnline();
}
public enum CodexLogLevel
@ -17,37 +17,39 @@
Error
}
public class OfflineCodexNode : IOfflineCodexNode
public class OfflineCodexNodes : IOfflineCodexNodes
{
private readonly IK8sManager k8SManager;
public int NumberOfNodes { get; }
public CodexLogLevel? LogLevel { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; private set; }
public ByteSize? StorageQuota { get; private set; }
public OfflineCodexNode(IK8sManager k8SManager)
public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
{
this.k8SManager = k8SManager;
NumberOfNodes = numberOfNodes;
}
public IOnlineCodexNode BringOnline()
public IOnlineCodexNodes BringOnline()
{
return k8SManager.BringOnline(this);
}
public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node)
public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
{
BootstrapNode = node;
return this;
}
public IOfflineCodexNode WithLogLevel(CodexLogLevel level)
public IOfflineCodexNodes WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota)
public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota)
{
StorageQuota = storageQuota;
return this;

16
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -7,25 +7,17 @@ namespace CodexDistTestCore
CodexDebugResponse GetDebugInfo();
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
IOfflineCodexNode BringOffline();
}
public class OnlineCodexNode : IOnlineCodexNode
{
private readonly IK8sManager k8SManager;
private readonly IFileManager fileManager;
private readonly int port;
private readonly CodexNodeContainer environment;
public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port)
public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
{
this.k8SManager = k8SManager;
this.fileManager = fileManager;
this.port = port;
}
public IOfflineCodexNode BringOffline()
{
return k8SManager.BringOffline(this);
this.environment = environment;
}
public CodexDebugResponse GetDebugInfo()
@ -55,7 +47,7 @@ namespace CodexDistTestCore
private Http Http()
{
return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1");
return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1");
}
}

33
CodexDistTestCore/OnlineCodexNodes.cs Normal file
View File

@ -0,0 +1,33 @@
namespace CodexDistTestCore
{
public interface IOnlineCodexNodes
{
IOfflineCodexNodes BringOffline();
IOnlineCodexNode this[int index] { get; }
}
public class OnlineCodexNodes : IOnlineCodexNodes
{
private readonly IK8sManager k8SManager;
private readonly IOnlineCodexNode[] nodes;
public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes)
{
this.k8SManager = k8SManager;
this.nodes = nodes;
}
public IOnlineCodexNode this[int index]
{
get
{
return nodes[index];
}
}
public IOfflineCodexNodes BringOffline()
{
return k8SManager.BringOffline(this);
}
}
}

4
LongTests/BasicTests/SimpleTests.cs
View File

@ -9,10 +9,10 @@ namespace LongTests.BasicTests
[Test, UseLongTimeouts]
public void OneClientLargeFileTest()
{
var primary = SetupCodexNode()
var primary = SetupCodexNodes(1)
.WithLogLevel(CodexLogLevel.Warn)
.WithStorageQuota(10.GB())
.BringOnline();
.BringOnline()[0];
var testFile = GenerateTestFile(1.GB());

6
Tests/BasicTests/SimpleTests.cs
View File

@ -11,7 +11,7 @@ namespace Tests.BasicTests
{
var dockerImage = new CodexDockerImage();
var node = SetupCodexNode().BringOnline();
var node = SetupCodexNodes(1).BringOnline()[0];
var debugInfo = node.GetDebugInfo();
@ -22,10 +22,10 @@ namespace Tests.BasicTests
[Test]
public void OneClientTest()
{
var primary = SetupCodexNode()
var primary = SetupCodexNodes(1)
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(2.MB())
.BringOnline();
.BringOnline()[0];
var testFile = GenerateTestFile(1.MB());