Adds WaitForCleanup test attribute to allow tests to wait for resources to be cleaned up

Framework/Core/EntryPoint.cs

@@ -38,10 +38,14 @@ namespace Core
             return new CoreInterface(this);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
         }
 
         internal T GetPlugin<T>() where T : IProjectPlugin

Framework/Core/PluginManager.cs

@@ -34,12 +34,12 @@
             return metadata;
         }
 
-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
             foreach (var pair in pairs)
             {
                 pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
             }
         }
 

Framework/Core/PluginTools.cs

@@ -7,7 +7,12 @@ namespace Core
     public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
     {
         ITimeSet TimeSet { get; }
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
     }
 
     public interface IWorkflowTool
@@ -73,9 +78,9 @@ namespace Core
             return workflowCreator.CreateWorkflow(namespaceOverride);
         }
 
-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
         {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
             if (deleteTrackedFiles) fileManager.DeleteAllFiles();
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -115,7 +115,7 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteAllNamespacesStartingWith(string prefix)
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
         {
             log.Debug();
 
@@ -124,25 +124,28 @@ namespace KubernetesWorkflow
 
             foreach (var ns in namespaces)
             {
-                DeleteNamespace(ns);
+                DeleteNamespace(ns, wait);
             }
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(K8sNamespace))
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
             }
         }
 
-        public void DeleteNamespace(string ns)
+        public void DeleteNamespace(string ns, bool wait)
         {
             log.Debug();
             if (IsNamespaceOnline(ns))
             {
                 client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
             }
         }
 
@@ -871,6 +874,11 @@ namespace KubernetesWorkflow
             WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
         }
 
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -17,8 +17,8 @@ namespace KubernetesWorkflow
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace();
+        void DeleteNamespace(bool wait);
-        void DeleteNamespacesStartingWith(string namespacePrefix);
+        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
     }
 
     public class StartupWorkflow : IStartupWorkflow
@@ -122,19 +122,19 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteNamespace();
+                controller.DeleteNamespace(wait);
             });
         }
 
-        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
         {
             K8s(controller =>
             {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
             });
         }
 

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -9,7 +9,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "27-D0-F6-EB-B9-A6-66-41-AA-EA-19-62-07-AF-47-41-25-5E-75-7E-97-35-CC-E1-C0-75-58-17-2D-87-11-75";
+        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-a518ec6-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
 
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -108,11 +108,11 @@ namespace CodexPlugin
 
     public class CodexSpace
     {
-        public int TotalBlocks { get; set; }
+        public long TotalBlocks { get; set; }
-        public int QuotaMaxBytes { get; set; }
+        public long QuotaMaxBytes { get; set; }
-        public int QuotaUsedBytes { get; set; }
+        public long QuotaUsedBytes { get; set; }
-        public int QuotaReservedBytes { get; set; }
+        public long QuotaReservedBytes { get; set; }
-        public int FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
+        public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
 
         public override string ToString()
         {

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -289,7 +289,7 @@ components:
           description: "Root hash of the content"
         originalBytes:
           type: integer
-          format: uint64
+          format: int64
           description: "Length of original content in bytes"
         blockSize:
           type: integer
@@ -304,18 +304,18 @@ components:
         totalBlocks:
           description: "Number of blocks stored by the node"
           type: integer
-          format: uint64
+          format: int64
         quotaMaxBytes:
           type: integer
-          format: uint64
+          format: int64
           description: "Maximum storage space used by the node"
         quotaUsedBytes:
           type: integer
-          format: uint64
+          format: int64
           description: "Amount of storage space currently in use"
         quotaReservedBytes:
           type: integer
-          format: uint64
+          format: int64
           description: "Amount of storage space reserved"
 
 servers:

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -148,7 +148,7 @@ namespace ContinuousTests
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
-            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
+            entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
         }
 
         private void PerformCleanup(ILog log)
@@ -157,7 +157,7 @@ namespace ContinuousTests
             log.Log("Cleaning up test namespace...");
 
             var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
-            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
+            entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
             log.Log("Cleanup finished.");
         }
     }

Tests/CodexContinuousTests/NodeRunner.cs

@@ -64,7 +64,7 @@ namespace ContinuousTests
             }
             finally
             {
-                entryPoint.Tools.CreateWorkflow().DeleteNamespace();
+                entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false);
             }
         }
 

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -54,7 +54,8 @@ namespace ContinuousTests
 
                     entryPoint.Decommission(
                         deleteKubernetesResources: false, // This would delete the continuous test net.
-                        deleteTrackedFiles: true
+                        deleteTrackedFiles: true,
+                        waitTillDone: false
                     );
                     runFinishedHandle.Set();
                 }

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -18,6 +18,7 @@ public class ScalabilityTests : CodexDistTest
     [Combinatorial]
     [UseLongTimeouts]
     [DontDownloadLogs]
+    [WaitForCleanup]
     public void ShouldMaintainFileInNetwork(
         [Values(10, 40)] int numberOfNodes, // TODO: include 80 and 100
         [Values(100, 1000, 5000, 10000)] int fileSizeInMb
@@ -64,6 +65,7 @@ public class ScalabilityTests : CodexDistTest
     [Combinatorial]
     [UseLongTimeouts]
     [DontDownloadLogs]
+    [WaitForCleanup]
     public void EveryoneGetsAFile(
         [Values(10, 40, 80, 100)] int numberOfNodes,
         [Values(100, 1000, 5000, 10000)] int fileSizeInMb

Tests/DistTestCore/DistTest.cs

@@ -52,7 +52,7 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
+                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
                 });                
             }
             catch (Exception ex)
@@ -72,7 +72,8 @@ namespace DistTestCore
             globalEntryPoint.Decommission(
                 // There shouldn't be any of either, but clean everything up regardless.
                 deleteKubernetesResources: true,
-                deleteTrackedFiles: true
+                deleteTrackedFiles: true,
+                waitTillDone: true
             );
         }
 
@@ -185,7 +186,13 @@ namespace DistTestCore
                 lock (lifecycleLock)
                 {
                     var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
+                    var lifecycle = new TestLifecycle(
+                        fixtureLog.CreateTestLog(),
+                        configuration,
+                        GetTimeSet(),
+                        testNamespace,
+                        deployId,
+                        ShouldWaitForCleanup());
                     lifecycles.Add(testName, lifecycle);
                     LifecycleStart(lifecycle);
                 }
@@ -235,6 +242,11 @@ namespace DistTestCore
             return new DefaultTimeSet();
         }
 
+        private bool ShouldWaitForCleanup()
+        {
+            return CurrentTestMethodHasAttribute<WaitForCleanupAttribute>();
+        }
+
         private bool ShouldUseLongTimeouts()
         {
             return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();

Tests/DistTestCore/TestLifecycle.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         private readonly List<RunningPod> runningContainers = new();
         private readonly string deployId;
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
         {
             Log = log;
             Configuration = configuration;
@@ -27,7 +27,7 @@ namespace DistTestCore
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
             this.deployId = deployId;
-
+            WaitForCleanup = waitForCleanup;
             log.WriteLogTag();
         }
 
@@ -35,13 +35,15 @@ namespace DistTestCore
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
 
         public void DeleteAllResources()
         {
             entryPoint.Decommission(
                 deleteKubernetesResources: true,
-                deleteTrackedFiles: true
+                deleteTrackedFiles: true,
+                waitTillDone: WaitForCleanup
             );
         }
 

Tests/DistTestCore/WaitForCleanupAttribute.cs

@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    /// <summary>
+    /// By default, test system does not wait until all resources are destroyed before starting the
+    /// next test. This saves a lot of time but it's not always what you want.
+    /// If you want to be sure the resources of your test are destroyed before the next test starts,
+    /// add this attribute to your test method.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class WaitForCleanupAttribute : PropertyAttribute
+    {
+    }
+}