From 1a31f324565b2fae1ea4f9781935322480d442c2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Mar 2023 12:35:03 +0100 Subject: [PATCH] Fixes timing issue in http retries --- CodexDistTestCore/Http.cs | 1 + CodexDistTestCore/K8sManager.cs | 14 ++--- CodexDistTestCore/K8sOperations.cs | 5 ++ CodexDistTestCore/Timing.cs | 10 ++-- .../{SimpleTests.cs => LargeFileTests.cs} | 17 ++---- LongTests/BasicTests/TestInfraTests.cs | 53 +++++++++++++++++++ 6 files changed, 75 insertions(+), 25 deletions(-) rename LongTests/BasicTests/{SimpleTests.cs => LargeFileTests.cs} (50%) create mode 100644 LongTests/BasicTests/TestInfraTests.cs diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs index 539877e..fd7e31a 100644 --- a/CodexDistTestCore/Http.cs +++ b/CodexDistTestCore/Http.cs @@ -79,6 +79,7 @@ namespace CodexDistTestCore } catch (Exception exception) { + Timing.HttpCallRetryDelay(); retryCounter++; if (retryCounter > Timing.HttpCallRetryCount()) { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index b410b3f..3530c96 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -24,7 +24,7 @@ { var online = CreateOnlineCodexNodes(offline); - K8s().BringOnline(online, offline); + K8s(k => k.BringOnline(online, offline)); log.Log($"{online.Describe()} online."); @@ -35,7 +35,7 @@ { var online = GetAndRemoveActiveNodeFor(node); - K8s().BringOffline(online); + K8s(k => k.BringOffline(online)); log.Log($"{online.Describe()} offline."); @@ -44,12 +44,12 @@ public void DeleteAllResources() { - K8s().DeleteAllResources(); + K8s(k => k.DeleteAllResources()); } public void FetchAllPodsLogs(IPodLogsHandler logHandler) { - K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); + K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler)); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) @@ -76,9 +76,11 @@ return n; } - private K8sOperations K8s() + private void K8s(Action action) { - return new K8sOperations(knownPods); + var k8s = new K8sOperations(knownPods); + action(k8s); + k8s.Close(); } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 2d65fd6..f9054b8 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -21,6 +21,11 @@ namespace CodexDistTestCore client = new Kubernetes(config); } + public void Close() + { + client.Dispose(); + } + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs index 26c49b2..7ee0edf 100644 --- a/CodexDistTestCore/Timing.cs +++ b/CodexDistTestCore/Timing.cs @@ -25,9 +25,9 @@ namespace CodexDistTestCore return GetTimes().HttpCallRetryCount(); } - public static void RetryDelay() + public static void HttpCallRetryDelay() { - Utils.Sleep(GetTimes().RetryDelay()); + Utils.Sleep(GetTimes().HttpCallRetryDelay()); } public static void WaitForK8sServiceDelay() @@ -52,7 +52,7 @@ namespace CodexDistTestCore { TimeSpan HttpCallTimeout(); int HttpCallRetryCount(); - TimeSpan RetryDelay(); + TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); } @@ -69,7 +69,7 @@ namespace CodexDistTestCore return 5; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromSeconds(3); } @@ -97,7 +97,7 @@ namespace CodexDistTestCore return 2; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromMinutes(5); } diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/LargeFileTests.cs similarity index 50% rename from LongTests/BasicTests/SimpleTests.cs rename to LongTests/BasicTests/LargeFileTests.cs index 44d1dca..b8ab20e 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -4,17 +4,17 @@ using NUnit.Framework; namespace LongTests.BasicTests { [TestFixture] - public class SimpleTests : DistTest + public class LargeFileTests : DistTest { [Test, UseLongTimeouts] public void OneClientLargeFileTest() { var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Warn) - .WithStorageQuota(10.GB()) + .WithStorageQuota(20.GB()) .BringOnline()[0]; - var testFile = GenerateTestFile(1.GB()); + var testFile = GenerateTestFile(10.GB()); var contentId = primary.UploadFile(testFile); @@ -22,16 +22,5 @@ namespace LongTests.BasicTests testFile.AssertIsEqual(downloadedFile); } - - [Test, UseLongTimeouts] - public void TestEnvironmentAddressSpaceTest() - { - var group = SetupCodexNodes(1000).BringOnline(); - - var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); - - Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), - "Not all created nodes provided a unique id."); - } } } diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs new file mode 100644 index 0000000..1e2f62e --- /dev/null +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -0,0 +1,53 @@ +using CodexDistTestCore; +using NUnit.Framework; + +namespace LongTests.BasicTests +{ + public class TestInfraTests : DistTest + { + [Test, UseLongTimeouts] + public void TestInfraShouldHave1000AddressSpacesPerPod() + { + var group = SetupCodexNodes(1000).BringOnline(); + + var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); + + Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), + "Not all created nodes provided a unique id."); + } + + [Test, UseLongTimeouts] + public void TestInfraSupportsManyConcurrentPods() + { + for (var i = 0; i < 20; i++) + { + var n = SetupCodexNodes(1).BringOnline()[0]; + + Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id)); + } + } + + [Test, UseLongTimeouts] + public void DownloadConsistencyTest() + { + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var files = new List(); + for (var i = 0; i < 100; i++) + { + files.Add(primary.DownloadContent(contentId)); + } + + Assert.That(files.All(f => f != null)); + Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize())); + foreach (var file in files) file!.AssertIsEqual(testFile); + } + } +}