diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index b60a11f7..d532fc6f 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -21,7 +21,7 @@ namespace DistTestCore public Logging.LogConfig GetLogConfig() { - return new Logging.LogConfig("CodexTestLogs", debugEnabled: false); + return new Logging.LogConfig("CodexTestLogs", debugEnabled: true); } public string GetFileManagerFolder() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 175fae60..ed96fc51 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -10,6 +10,7 @@ using System.Reflection; namespace DistTestCore { [SetUpFixture] + [Parallelizable(ParallelScope.All)] public abstract class DistTest { private readonly Configuration configuration = new Configuration(); diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs index a0455005..daf8b9b0 100644 --- a/KubernetesWorkflow/CommandRunner.cs +++ b/KubernetesWorkflow/CommandRunner.cs @@ -5,7 +5,7 @@ namespace KubernetesWorkflow { public class CommandRunner { - private readonly Kubernetes client; + private readonly K8sClient client; private readonly string k8sNamespace; private readonly RunningPod pod; private readonly string containerName; @@ -13,7 +13,7 @@ namespace KubernetesWorkflow private readonly string[] arguments; private readonly List lines = new List(); - public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) + public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) { this.client = client; this.k8sNamespace = k8sNamespace; @@ -27,8 +27,8 @@ namespace KubernetesWorkflow { var input = new[] { command }.Concat(arguments).ToArray(); - Time.Wait(client.NamespacedPodExecAsync( - pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); + Time.Wait(client.Run(c => c.NamespacedPodExecAsync( + pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); } public string GetStdOut() diff --git a/KubernetesWorkflow/K8sClient.cs b/KubernetesWorkflow/K8sClient.cs new file mode 100644 index 00000000..14b53ce4 --- /dev/null +++ b/KubernetesWorkflow/K8sClient.cs @@ -0,0 +1,36 @@ +using k8s; + +namespace KubernetesWorkflow +{ + public class K8sClient + { + private readonly Kubernetes client; + private static readonly object clientLock = new object(); + + public K8sClient(KubernetesClientConfiguration config) + { + client = new Kubernetes(config); + } + + public void Run(Action action) + { + lock (clientLock) + { + action(client); + } + } + + public T Run(Func action) + { + lock (clientLock) + { + return action(client); + } + } + + public void Dispose() + { + client.Dispose(); + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 118101d8..d50ac28c 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,8 +11,7 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; - private readonly string testNamespace; - private readonly Kubernetes client; + private readonly K8sClient client; public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) { @@ -20,8 +19,10 @@ namespace KubernetesWorkflow this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; - this.testNamespace = testNamespace; - client = new Kubernetes(cluster.GetK8sClientConfig()); + client = new K8sClient(cluster.GetK8sClientConfig()); + + K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; + log.Debug($"'{K8sTestNamespace}'"); } public void Dispose() @@ -53,7 +54,7 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name)); logHandler.Log(stream); } @@ -69,7 +70,7 @@ namespace KubernetesWorkflow { log.Debug(); - var all = client.ListNamespace().Items; + var all = client.Run(c => c.ListNamespace().Items); var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix)); foreach (var ns in namespaces) @@ -87,7 +88,7 @@ namespace KubernetesWorkflow log.Debug(); if (IsTestNamespaceOnline()) { - client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0); + client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0)); } WaitUntilNamespaceDeleted(); } @@ -97,12 +98,14 @@ namespace KubernetesWorkflow log.Debug(); if (IsNamespaceOnline(ns)) { - client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0); + client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0)); } } #region Namespace management + private string K8sTestNamespace { get; } + private void EnsureTestNamespace() { if (IsTestNamespaceOnline()) return; @@ -116,15 +119,10 @@ namespace KubernetesWorkflow Labels = new Dictionary { { "name", K8sTestNamespace } } } }; - client.CreateNamespace(namespaceSpec); + client.Run(c => c.CreateNamespace(namespaceSpec)); WaitUntilNamespaceCreated(); } - private string K8sTestNamespace - { - get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; } - } - private bool IsTestNamespaceOnline() { return IsNamespaceOnline(K8sTestNamespace); @@ -132,7 +130,7 @@ namespace KubernetesWorkflow private bool IsNamespaceOnline(string name) { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == name); + return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name)); } #endregion @@ -167,7 +165,7 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace); + client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace)); WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); return deploymentSpec.Metadata.Name; @@ -175,7 +173,7 @@ namespace KubernetesWorkflow private void DeleteDeployment(string deploymentName) { - client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace); + client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace)); WaitUntilDeploymentOffline(deploymentName); } @@ -283,14 +281,14 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedService(serviceSpec, K8sTestNamespace); + client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace)); return (serviceSpec.Metadata.Name, result); } private void DeleteService(string serviceName) { - client.DeleteNamespacedService(serviceName, K8sTestNamespace); + client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace)); } private V1ObjectMeta CreateServiceMetadata() @@ -358,7 +356,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace); + var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }); } @@ -367,7 +365,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployments = client.ListNamespacedDeployment(K8sTestNamespace); + var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace)); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; }); @@ -377,7 +375,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var pods = client.ListNamespacedPod(K8sTestNamespace).Items; + var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); return pod == null; }); @@ -400,7 +398,7 @@ namespace KubernetesWorkflow private (string, string) FetchNewPod() { - var pods = client.ListNamespacedPod(K8sTestNamespace).Items; + var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); diff --git a/Tests/BasicTests/DownloadTests.cs b/Tests/BasicTests/DownloadTests.cs index 0f61daf7..44ea4ee2 100644 --- a/Tests/BasicTests/DownloadTests.cs +++ b/Tests/BasicTests/DownloadTests.cs @@ -6,6 +6,7 @@ namespace Tests.ParallelTests [TestFixture] public class DownloadTests : DistTest { + [Ignore("a")] [TestCase(3, 500)] [TestCase(5, 100)] [TestCase(10, 256)] diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs new file mode 100644 index 00000000..b26dcbe8 --- /dev/null +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -0,0 +1,51 @@ +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [Ignore("not a real test!")] + [TestFixture] + public class NetworkIsolationTest : DistTest + { + private IOnlineCodexNode? node = null; + + // net isolation: only on real cluster? + // parallel upload/download tests? + // operation times. + + [Test] + public void SetUpANodeAndWait() + { + node = SetupCodexNode(); + + while (node != null) + { + Time.Sleep(TimeSpan.FromSeconds(5)); + } + } + + [Test] + public void ForeignNodeConnects() + { + var myNode = SetupCodexNode(); + + while (node == null) + { + Time.Sleep(TimeSpan.FromSeconds(1)); + } + + myNode.ConnectToPeer(node); + + var testFile = GenerateTestFile(1.MB()); + + var contentId = node.UploadFile(testFile); + + var downloadedFile = myNode.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + + node = null; + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 14e0e215..461dfa85 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -5,18 +5,25 @@ using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] - [Parallelizable(ParallelScope.All)] public class TwoClientTests : DistTest { - [Test] - public void TwoClientsOnePodTest() + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + [TestCase(4)] + [TestCase(5)] + [TestCase(6)] + [TestCase(7)] + [TestCase(8)] + [TestCase(9)] + public void TwoClientsOnePodTest(int size) { var group = SetupCodexNodes(2); var primary = group[0]; var secondary = group[1]; - PerformTwoClientTest(primary, secondary); + PerformTwoClientTest(primary, secondary, size.MB()); } [Test] @@ -39,10 +46,15 @@ namespace Tests.BasicTests } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + PerformTwoClientTest(primary, secondary, 1.MB()); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size) { primary.ConnectToPeer(secondary); - var testFile = GenerateTestFile(1.MB()); + var testFile = GenerateTestFile(size); var contentId = primary.UploadFile(testFile); diff --git a/Tests/BasicTests/UploadTests.cs b/Tests/BasicTests/UploadTests.cs index 8cbbe7a1..fbe146f4 100644 --- a/Tests/BasicTests/UploadTests.cs +++ b/Tests/BasicTests/UploadTests.cs @@ -6,6 +6,7 @@ namespace Tests.ParallelTests [TestFixture] public class UploadTests : DistTest { + [Ignore("a")] [TestCase(3, 50)] [TestCase(5, 75)] [TestCase(10, 25)] diff --git a/Tests/Parallelism.cs b/Tests/Parallelism.cs new file mode 100644 index 00000000..c6f80894 --- /dev/null +++ b/Tests/Parallelism.cs @@ -0,0 +1,6 @@ +using NUnit.Framework; + +[assembly: LevelOfParallelism(30)] +namespace Tests +{ +}