Fixes timing issue in http retries

CodexDistTestCore/Http.cs

@@ -79,6 +79,7 @@ namespace CodexDistTestCore
                 }
                 catch (Exception exception)
                 {
+                    Timing.HttpCallRetryDelay();
                     retryCounter++;
                     if (retryCounter > Timing.HttpCallRetryCount())
                     {

CodexDistTestCore/K8sManager.cs

@@ -24,7 +24,7 @@
         {
             var online = CreateOnlineCodexNodes(offline);
 
-            K8s().BringOnline(online, offline);
+            K8s(k => k.BringOnline(online, offline));
 
             log.Log($"{online.Describe()} online.");
 
@@ -35,7 +35,7 @@
         {
             var online = GetAndRemoveActiveNodeFor(node);
 
-            K8s().BringOffline(online);
+            K8s(k => k.BringOffline(online));
 
             log.Log($"{online.Describe()} offline.");
 
@@ -44,12 +44,12 @@
 
         public void DeleteAllResources()
         {
-            K8s().DeleteAllResources();
+            K8s(k => k.DeleteAllResources());
         }
 
         public void FetchAllPodsLogs(IPodLogsHandler logHandler)
         {
-            K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler);
+            K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler));
         }
 
         private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
@@ -76,9 +76,11 @@
             return n;
         }
 
-        private K8sOperations K8s()
+        private void K8s(Action<K8sOperations> action)
         {
-            return new K8sOperations(knownPods);
+            var k8s = new K8sOperations(knownPods);
+            action(k8s);
+            k8s.Close();
         }
     }
 }

CodexDistTestCore/K8sOperations.cs

@@ -21,6 +21,11 @@ namespace CodexDistTestCore
             client = new Kubernetes(config);
         }
 
+        public void Close()
+        {
+            client.Dispose();
+        }
+
         public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
         {
             EnsureTestNamespace();

CodexDistTestCore/Timing.cs

@@ -25,9 +25,9 @@ namespace CodexDistTestCore
             return GetTimes().HttpCallRetryCount();
         }
 
-        public static void RetryDelay()
+        public static void HttpCallRetryDelay()
         {
-            Utils.Sleep(GetTimes().RetryDelay());
+            Utils.Sleep(GetTimes().HttpCallRetryDelay());
         }
 
         public static void WaitForK8sServiceDelay()
@@ -52,7 +52,7 @@ namespace CodexDistTestCore
     {
         TimeSpan HttpCallTimeout();
         int HttpCallRetryCount();
-        TimeSpan RetryDelay();
+        TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
     }
@@ -69,7 +69,7 @@ namespace CodexDistTestCore
             return 5;
         }
 
-        public TimeSpan RetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
             return TimeSpan.FromSeconds(3);
         }
@@ -97,7 +97,7 @@ namespace CodexDistTestCore
             return 2;
         }
 
-        public TimeSpan RetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
             return TimeSpan.FromMinutes(5);
         }

LongTests/BasicTests/LargeFileTests.cs

@@ -4,17 +4,17 @@ using NUnit.Framework;
 namespace LongTests.BasicTests
 {
     [TestFixture]
-    public class SimpleTests : DistTest
+    public class LargeFileTests : DistTest
     {
         [Test, UseLongTimeouts]
         public void OneClientLargeFileTest()
         {
             var primary = SetupCodexNodes(1)
                                 .WithLogLevel(CodexLogLevel.Warn)
-                                .WithStorageQuota(10.GB())
+                                .WithStorageQuota(20.GB())
                                 .BringOnline()[0];
 
-            var testFile = GenerateTestFile(1.GB());
+            var testFile = GenerateTestFile(10.GB());
 
             var contentId = primary.UploadFile(testFile);
 
@@ -22,16 +22,5 @@ namespace LongTests.BasicTests
 
             testFile.AssertIsEqual(downloadedFile);
         }
-
-        [Test, UseLongTimeouts]
-        public void TestEnvironmentAddressSpaceTest()
-        {
-            var group = SetupCodexNodes(1000).BringOnline();
-
-            var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();
-
-            Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()),
-                "Not all created nodes provided a unique id.");
-        }
     }
 }

LongTests/BasicTests/TestInfraTests.cs

@@ -0,0 +1,53 @@
+using CodexDistTestCore;
+using NUnit.Framework;
+
+namespace LongTests.BasicTests
+{
+    public class TestInfraTests : DistTest
+    {
+        [Test, UseLongTimeouts]
+        public void TestInfraShouldHave1000AddressSpacesPerPod()
+        {
+            var group = SetupCodexNodes(1000).BringOnline();
+
+            var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();
+
+            Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()),
+                "Not all created nodes provided a unique id.");
+        }
+
+        [Test, UseLongTimeouts]
+        public void TestInfraSupportsManyConcurrentPods()
+        {
+            for (var i = 0; i < 20; i++)
+            {
+                var n = SetupCodexNodes(1).BringOnline()[0];
+
+                Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id));
+            }
+        }
+
+        [Test, UseLongTimeouts]
+        public void DownloadConsistencyTest()
+        {
+            var primary = SetupCodexNodes(1)
+                            .WithLogLevel(CodexLogLevel.Trace)
+                            .WithStorageQuota(2.MB())
+                            .BringOnline()[0];
+
+            var testFile = GenerateTestFile(1.MB());
+
+            var contentId = primary.UploadFile(testFile);
+
+            var files = new List<TestFile?>();
+            for (var i = 0; i < 100; i++)
+            {
+                files.Add(primary.DownloadContent(contentId));
+            }
+
+            Assert.That(files.All(f => f != null));
+            Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize()));
+            foreach (var file in files) file!.AssertIsEqual(testFile);
+        }
+    }
+}