From eae138f7fda9597e8528bebc4bd95ffb82deab24 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 4 May 2023 14:55:39 +0200 Subject: [PATCH] Adds policy and test for network isolation. This will not work on docker-desktop clusters. --- DistTestCore/DistTest.cs | 11 +++- DistTestCore/OnlineCodexNode.cs | 4 ++ KubernetesWorkflow/K8sController.cs | 73 +++++++++++++++++++++++- Tests/BasicTests/NetworkIsolationTest.cs | 36 ++++++------ 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 2629973..e5ee885 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -128,9 +128,16 @@ namespace DistTestCore return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } - protected BaseLog Log + protected void Log(string msg) { - get { return Get().Log; } + TestContext.Progress.WriteLine(msg); + Get().Log.Log(msg); + } + + protected void Debug(string msg) + { + TestContext.Progress.WriteLine(msg); + Get().Log.Debug(msg); } private TestLifecycle Get() diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 91cf11a..7fb30e1 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -92,6 +92,10 @@ namespace DistTestCore public ICodexSetup BringOffline() { + if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + + "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + + "available for codex-nodes in groups of 1."); + return Group.BringOffline(); } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d50ac28..fd42d07 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -22,7 +22,7 @@ namespace KubernetesWorkflow client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; - log.Debug($"'{K8sTestNamespace}'"); + log.Debug($"Test namespace: '{K8sTestNamespace}'"); } public void Dispose() @@ -121,6 +121,8 @@ namespace KubernetesWorkflow }; client.Run(c => c.CreateNamespace(namespaceSpec)); WaitUntilNamespaceCreated(); + + CreatePolicy(); } private bool IsTestNamespaceOnline() @@ -133,6 +135,67 @@ namespace KubernetesWorkflow return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name)); } + private void CreatePolicy() + { + client.Run(c => + { + var body = new V1NetworkPolicy + { + Metadata = new V1ObjectMeta + { + Name = "isolate-policy", + NamespaceProperty = K8sTestNamespace + }, + Spec = new V1NetworkPolicySpec + { + PodSelector = new V1LabelSelector + { + MatchLabels = GetSelector() + }, + PolicyTypes = new[] + { + "Ingress", + "Egress" + }, + Ingress = new List + { + new V1NetworkPolicyIngressRule + { + FromProperty = new List + { + new V1NetworkPolicyPeer + { + NamespaceSelector = new V1LabelSelector + { + MatchLabels = GetMyNamespaceSelector() + } + } + } + } + }, + Egress = new List + { + new V1NetworkPolicyEgressRule + { + To = new List + { + new V1NetworkPolicyPeer + { + NamespaceSelector = new V1LabelSelector + { + MatchLabels = GetMyNamespaceSelector() + } + } + } + } + } + } + }; + + c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace); + }); + } + #endregion #region Deployment management @@ -192,12 +255,18 @@ namespace KubernetesWorkflow return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; } + private IDictionary GetMyNamespaceSelector() + { + return new Dictionary { { "name", "thatisincorrect" } }; + } + private V1ObjectMeta CreateDeploymentMetadata() { return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sTestNamespace + NamespaceProperty = K8sTestNamespace, + Labels = GetSelector() }; } diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs index a7b2520..f3406d4 100644 --- a/Tests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -4,23 +4,20 @@ using Utils; namespace Tests.BasicTests { - [Ignore("not a real test!")] + // Warning! + // This is a test to check network-isolation in the test-infrastructure. + // It requires parallelism(2) or greater to run. [TestFixture] public class NetworkIsolationTest : DistTest { private IOnlineCodexNode? node = null; - // net isolation: only on real cluster? - [Test] public void SetUpANodeAndWait() { node = SetupCodexNode(); - while (node != null) - { - Time.Sleep(TimeSpan.FromSeconds(5)); - } + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); } [Test] @@ -28,22 +25,21 @@ namespace Tests.BasicTests { var myNode = SetupCodexNode(); - while (node == null) + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + + try { - Time.Sleep(TimeSpan.FromSeconds(1)); + myNode.ConnectToPeer(node!); + } + catch + { + // Good! This connection should be prohibited by the network isolation policy. + node = null; + return; } - myNode.ConnectToPeer(node); - - var testFile = GenerateTestFile(1.MB()); - - var contentId = node.UploadFile(testFile); - - var downloadedFile = myNode.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - - node = null; + Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " + + "This may cause cross-test interference. Network isolation policy should be applied. Test infra failure."); } } }