codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup and split up for test supporting methods

This commit is contained in:
benbierens 2023-03-19 10:49:03 +01:00
parent b4ab5798f3
commit 457951c561
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
6 changed files with 550 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
BasicTests/DebugEndpointTests.cs
View File

@ -9,14 +9,35 @@ namespace CodexDistTests.BasicTests
[Test]
public void GetDebugInfo()
{
CreateCodexNode();
var node = SetupCodexNode().BringOnline();
var node = GetCodexNode();
var debugInfo = node.GetDebugInfo();
Assert.That(debugInfo.spr, Is.Not.Empty);
DestroyCodexNode();
}
//[Test]
//public void TwoClientTest()
//{
// var primaryNodex = SetupCodexNode()
// .WithLogLevel(CodexLogLevel.Warn)
// .WithStorageQuota(1024 * 1024)
// .BringOnline();
// var secondaryNodex = SetupCodexNode()
// .WithBootstrapNode(primaryNodex)
// .BringOnline();
// var testFile = GenerateTestFile(1024 * 1024);
// var contentId = primaryNodex.UploadFile(testFile);
// var downloadedFile = secondaryNodex.DownloadContent(contentId);
// testFile.AssertIsEqual(downloadedFile);
// // Test files are automatically deleted.
// // Online nodes are automatically destroyed.
//}
}
}

139
TestCore/DistTest.cs
View File

@ -1,139 +1,34 @@
using k8s;
using k8s.Models;
using NUnit.Framework;
namespace CodexDistTests.TestCore
{
public abstract class DistTest
{
private const string k8sNamespace = "codex-test-namespace";
private FileManager fileManager = null!;
private K8sManager k8sManager = null!;
private V1Namespace? activeNamespace;
private V1Deployment? activeDeployment;
private V1Service? activeService;
public void CreateCodexNode()
[SetUp]
public void SetUpDistTest()
{
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
var client = new Kubernetes(config);
var namespaceSpec = new V1Namespace
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = k8sNamespace,
Labels = new Dictionary<string, string> { { "name", k8sNamespace } }
}
};
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = "codex-demo",
NamespaceProperty = k8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = new Dictionary<string, string> { { "codex-node", "dist-test" } }
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = new Dictionary<string, string> { { "codex-node", "dist-test" } }
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = "codex-node",
Image = "thatbenbierens/nim-codex:sha-c9a62de",
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 8080,
Name = "codex-api-port"
}
},
Env = new List<V1EnvVar>
{
new V1EnvVar
{
Name = "LOG_LEVEL",
Value = "WARN"
}
}
}
}
}
}
}
};
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = "codex-entrypoint",
NamespaceProperty = k8sNamespace
},
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = new Dictionary<string, string> { { "codex-node", "dist-test" } },
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Protocol = "TCP",
Port = 8080,
TargetPort = "codex-api-port",
NodePort = 30001
}
}
}
};
activeNamespace = client.CreateNamespace(namespaceSpec);
activeDeployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace);
activeService = client.CreateNamespacedService(serviceSpec, k8sNamespace);
// todo: wait until online!
while (activeDeployment.Status.AvailableReplicas == null || activeDeployment.Status.AvailableReplicas != 1)
{
Timing.WaitForServiceDelay();
activeDeployment = client.ReadNamespacedDeployment(activeDeployment.Name(), k8sNamespace);
}
fileManager = new FileManager();
k8sManager = new K8sManager(fileManager);
}
public CodexNode GetCodexNode()
[TearDown]
public void TearDownDistTest()
{
return new CodexNode(30001); // matches service spec.
fileManager.DeleteAllTestFiles();
k8sManager.DeleteAllResources();
}
public void DestroyCodexNode()
public TestFile GenerateTestFile(int size = 1024)
{
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
var client = new Kubernetes(config);
return fileManager.GenerateTestFile(size);
}
client.DeleteNamespacedService(activeService.Name(), k8sNamespace);
client.DeleteNamespacedDeployment(activeDeployment.Name(), k8sNamespace);
client.DeleteNamespace(activeNamespace.Name());
// todo: wait until terminated!
var pods = client.ListNamespacedPod(k8sNamespace);
while (pods.Items.Any())
{
Timing.WaitForServiceDelay();
pods = client.ListNamespacedPod(k8sNamespace);
}
public IOfflineCodexNode SetupCodexNode()
{
return new OfflineCodexNode(k8sManager);
}
}
}

92
TestCore/FileManager.cs Normal file
View File

@ -0,0 +1,92 @@
using NUnit.Framework;
namespace CodexDistTests.TestCore
{
public interface IFileManager
{
TestFile CreateEmptyTestFile();
TestFile GenerateTestFile(int size = 1024);
void DeleteAllTestFiles();
}
public class FileManager : IFileManager
{
public const int ChunkSize = 1024;
private const string Folder = "TestDataFiles";
private readonly Random random = new Random();
private readonly List<TestFile> activeFiles = new List<TestFile>();
public TestFile CreateEmptyTestFile()
{
var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
activeFiles.Add(result);
return result;
}
public TestFile GenerateTestFile(int size = 1024)
{
var result = CreateEmptyTestFile();
GenerateFileBytes(result, size);
return result;
}
public void DeleteAllTestFiles()
{
foreach (var file in activeFiles) File.Delete(file.Filename);
activeFiles.Clear();
}
private void GenerateFileBytes(TestFile result, int size)
{
while (size > 0)
{
var length = Math.Min(size, ChunkSize);
AppendRandomBytesToFile(result, length);
size -= length;
}
}
private void AppendRandomBytesToFile(TestFile result, int length)
{
var bytes = new byte[length];
random.NextBytes(bytes);
using var stream = new FileStream(result.Filename, FileMode.Append);
stream.Write(bytes, 0, bytes.Length);
}
}
public class TestFile
{
public TestFile(string filename)
{
Filename = filename;
}
public string Filename { get; }
public void AssertIsEqual(TestFile? other)
{
if (other == null) Assert.Fail("TestFile is null.");
if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);
var bytes1 = new byte[FileManager.ChunkSize];
var bytes2 = new byte[FileManager.ChunkSize];
var read1 = 0;
var read2 = 0;
while (true)
{
read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize);
read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize);
if (read1 == 0 && read2 == 0) return;
Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length.");
CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal.");
}
}
}
}

323
TestCore/K8sManager.cs Normal file
View File

@ -0,0 +1,323 @@
using k8s;
using k8s.Models;
namespace CodexDistTests.TestCore
{
public interface IK8sManager
{
IOnlineCodexNode BringOnline(OfflineCodexNode node);
}
public class K8sManager : IK8sManager
{
private const string k8sNamespace = "codex-test-namespace";
private const string codexDockerImage = "thatbenbierens/nim-codex:sha-c9a62de";
private readonly IFileManager fileManager;
private int freePort;
private int nodeOrderNumber;
private V1Namespace? activeNamespace;
private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
public K8sManager(IFileManager fileManager)
{
this.fileManager = fileManager;
freePort = 30001;
nodeOrderNumber = 0;
}
public IOnlineCodexNode BringOnline(OfflineCodexNode node)
{
var client = CreateClient();
EnsureTestNamespace(client);
var activeNode = new ActiveNode(GetFreePort(), GetNodeOrderNumber());
var codexNode = new OnlineCodexNode(node, fileManager, activeNode.Port);
activeNodes.Add(codexNode, activeNode);
CreateDeployment(activeNode, client);
CreateService(activeNode, client);
WaitUntilOnline(activeNode, client);
return codexNode;
}
public IOfflineCodexNode BringOffline(IOnlineCodexNode node)
{
var client = CreateClient();
var n = (OnlineCodexNode)node;
var activeNode = activeNodes[n];
activeNodes.Remove(n);
var deploymentName = activeNode.Deployment.Name();
BringOffline(activeNode, client);
WaitUntilOffline(deploymentName, client);
return n.Origin;
}
public void DeleteAllResources()
{
var client = CreateClient();
foreach (var activeNode in activeNodes.Values)
{
BringOffline(activeNode, client);
}
DeleteNamespace(client);
WaitUntilZeroPods(client);
WaitUntilNamespaceDeleted(client);
}
private void BringOffline(ActiveNode activeNode, Kubernetes client)
{
DeleteDeployment(activeNode, client);
DeleteService(activeNode, client);
}
#region Waiting
private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
{
while (activeNode.Deployment?.Status.AvailableReplicas == null || activeNode.Deployment.Status.AvailableReplicas != 1)
{
Timing.WaitForServiceDelay();
activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace);
}
}
private void WaitUntilOffline(string deploymentName, Kubernetes client)
{
var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
while (deployment != null && deployment.Status.AvailableReplicas > 0)
{
Timing.WaitForServiceDelay();
deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
}
}
private void WaitUntilZeroPods(Kubernetes client)
{
var pods = client.ListNamespacedPod(k8sNamespace);
while (pods.Items.Any())
{
Timing.WaitForServiceDelay();
pods = client.ListNamespacedPod(k8sNamespace);
}
}
private void WaitUntilNamespaceDeleted(Kubernetes client)
{
var namespaces = client.ListNamespace();
while (namespaces.Items.Any(n => n.Metadata.Name == k8sNamespace))
{
Timing.WaitForServiceDelay();
namespaces = client.ListNamespace();
}
}
#endregion
#region Service management
private void CreateService(ActiveNode node, Kubernetes client)
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = node.GetServiceMetadata(),
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = node.GetSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Protocol = "TCP",
Port = 8080,
TargetPort = node.GetContainerPortName(),
NodePort = node.Port
}
}
}
};
node.Service = client.CreateNamespacedService(serviceSpec, k8sNamespace);
}
private void DeleteService(ActiveNode node, Kubernetes client)
{
if (node.Service == null) return;
client.DeleteNamespacedService(node.Service.Name(), k8sNamespace);
node.Service = null;
}
#endregion
#region Deployment management
private void CreateDeployment(ActiveNode node, Kubernetes client)
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = node.GetDeploymentMetadata(),
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = node.GetSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = node.GetSelector()
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = node.GetContainerName(),
Image = codexDockerImage,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 8080,
Name = node.GetContainerPortName()
}
},
Env = new List<V1EnvVar>// todo
{
new V1EnvVar
{
Name = "LOG_LEVEL",
Value = "WARN"
}
}
}
}
}
}
}
};
node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace);
}
private void DeleteDeployment(ActiveNode node, Kubernetes client)
{
if (node.Deployment == null) return;
client.DeleteNamespacedDeployment(node.Deployment.Name(), k8sNamespace);
node.Deployment = null;
}
#endregion
#region Namespace management
private void EnsureTestNamespace(Kubernetes client)
{
if (activeNamespace != null) return;
var namespaceSpec = new V1Namespace
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = k8sNamespace,
Labels = new Dictionary<string, string> { { "name", k8sNamespace } }
}
};
activeNamespace = client.CreateNamespace(namespaceSpec);
}
private void DeleteNamespace(Kubernetes client)
{
if (activeNamespace == null) return;
client.DeleteNamespace(activeNamespace.Name());
}
#endregion
private static Kubernetes CreateClient()
{
// todo: If the default KubeConfig file does not suffice, change it here:
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
return new Kubernetes(config);
}
private int GetFreePort()
{
var port = freePort;
freePort++;
return port;
}
private int GetNodeOrderNumber()
{
var number = nodeOrderNumber;
nodeOrderNumber++;
return number;
}
public class ActiveNode
{
public ActiveNode(int port, int orderNumber)
{
SelectorName = orderNumber.ToString().PadLeft(6, '0');
Port = port;
}
public string SelectorName { get; }
public int Port { get; }
public V1Deployment? Deployment { get; set; }
public V1Service? Service { get; set; }
public V1ObjectMeta GetServiceMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + SelectorName,
NamespaceProperty = k8sNamespace
};
}
public V1ObjectMeta GetDeploymentMetadata()
{
return new V1ObjectMeta
{
Name = "codex-test-node-" + SelectorName,
NamespaceProperty = k8sNamespace
};
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
}
public string GetContainerPortName()
{
//Caution, was: "codex-api-port" + SelectorName
//but string length causes 'UnprocessableEntity' exception in k8s.
return "api-" + SelectorName;
}
public string GetContainerName()
{
return "codex-test-node";
}
}
}
}

56
TestCore/OfflineCodexNode.cs Normal file
View File

@ -0,0 +1,56 @@
namespace CodexDistTests.TestCore
{
public interface IOfflineCodexNode
{
IOfflineCodexNode WithLogLevel(CodexLogLevel level);
IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNode WithStorageQuota(int storageQuotaBytes);
IOnlineCodexNode BringOnline();
}
public enum CodexLogLevel
{
Trace,
Debug,
Info,
Warn,
Error
}
public class OfflineCodexNode : IOfflineCodexNode
{
private readonly IK8sManager k8SManager;
public CodexLogLevel? LogLevel { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; private set; }
public int? StorageQuota { get; private set; }
public OfflineCodexNode(IK8sManager k8SManager)
{
this.k8SManager = k8SManager;
}
public IOnlineCodexNode BringOnline()
{
return k8SManager.BringOnline(this);
}
public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node)
{
BootstrapNode = node;
return this;
}
public IOfflineCodexNode WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public IOfflineCodexNode WithStorageQuota(int storageQuotaBytes)
{
StorageQuota = storageQuotaBytes;
return this;
}
}
}

45
TestCore/CodexNode.cs → TestCore/OnlineCodexNode.cs
View File

@ -4,12 +4,24 @@ using System.Net.Http.Headers;
namespace CodexDistTests.TestCore
{
public class CodexNode
public interface IOnlineCodexNode
{
CodexDebugResponse GetDebugInfo();
ContentId UploadFile(TestFile file, int retryCounter = 0);
TestFile? DownloadContent(ContentId contentId);
}
public class OnlineCodexNode : IOnlineCodexNode
{
private readonly IFileManager fileManager;
private readonly int port;
public CodexNode(int port)
public OfflineCodexNode Origin { get; }
public OnlineCodexNode(OfflineCodexNode origin, IFileManager fileManager, int port)
{
Origin = origin;
this.fileManager = fileManager;
this.port = port;
}
@ -18,20 +30,21 @@ namespace CodexDistTests.TestCore
return HttpGet<CodexDebugResponse>("debug/info");
}
public string UploadFile(string filename, int retryCounter = 0)
public ContentId UploadFile(TestFile file, int retryCounter = 0)
{
try
{
var url = $"http://127.0.0.1:{port}/api/codex/v1/upload";
using var client = GetClient();
var byteData = File.ReadAllBytes(filename);
// Todo: If the file is too large to read into memory, we'll need to rewrite this upload POST to be streaming.
var byteData = File.ReadAllBytes(file.Filename);
using var content = new ByteArrayContent(byteData);
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = Utils.Wait(client.PostAsync(url, content));
var contentId = Utils.Wait(response.Content.ReadAsStringAsync());
return contentId;
return new ContentId(contentId);
}
catch (Exception exception)
{
@ -43,14 +56,20 @@ namespace CodexDistTests.TestCore
else
{
Timing.RetryDelay();
return UploadFile(filename, retryCounter + 1);
return UploadFile(file, retryCounter + 1);
}
}
}
public byte[]? DownloadContent(string contentId)
public TestFile? DownloadContent(ContentId contentId)
{
return HttpGetBytes("download/" + contentId);
// Todo: If the file is too large, rewrite to streaming:
var bytes = HttpGetBytes("download/" + contentId.Id);
if (bytes == null) return null;
var file = fileManager.CreateEmptyTestFile();
File.WriteAllBytes(file.Filename, bytes);
return file;
}
private byte[]? HttpGetBytes(string endpoint, int retryCounter = 0)
@ -124,4 +143,14 @@ namespace CodexDistTests.TestCore
public string version { get; set; } = string.Empty;
public string revision { get; set; } = string.Empty;
}
public class ContentId
{
public ContentId(string id)
{
Id = id;
}
public string Id { get; }
}
}