diff --git a/Framework/Core/EntryPoint.cs b/Framework/Core/EntryPoint.cs index 7977eb3..0db40f3 100644 --- a/Framework/Core/EntryPoint.cs +++ b/Framework/Core/EntryPoint.cs @@ -38,10 +38,14 @@ namespace Core return new CoreInterface(this); } - public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles) + /// + /// Deletes kubernetes and tracked file resources. + /// when `waitTillDone` is true, this function will block until resources are deleted. + /// + public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone) { - manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles); - Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles); + manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone); + Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone); } internal T GetPlugin() where T : IProjectPlugin diff --git a/Framework/Core/PluginManager.cs b/Framework/Core/PluginManager.cs index e2b2a5c..27b08fe 100644 --- a/Framework/Core/PluginManager.cs +++ b/Framework/Core/PluginManager.cs @@ -34,12 +34,12 @@ return metadata; } - internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles) + internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone) { foreach (var pair in pairs) { pair.Plugin.Decommission(); - pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles); + pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone); } } diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs index 78f7814..aa8e0a6 100644 --- a/Framework/Core/PluginTools.cs +++ b/Framework/Core/PluginTools.cs @@ -7,7 +7,12 @@ namespace Core public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool { ITimeSet TimeSet { get; } - void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles); + + /// + /// Deletes kubernetes and tracked file resources. + /// when `waitTillDone` is true, this function will block until resources are deleted. + /// + void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone); } public interface IWorkflowTool @@ -73,9 +78,9 @@ namespace Core return workflowCreator.CreateWorkflow(namespaceOverride); } - public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles) + public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone) { - if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(); + if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone); if (deleteTrackedFiles) fileManager.DeleteAllFiles(); } diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 0f50d87..30de0c9 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -115,7 +115,7 @@ namespace KubernetesWorkflow }); } - public void DeleteAllNamespacesStartingWith(string prefix) + public void DeleteAllNamespacesStartingWith(string prefix, bool wait) { log.Debug(); @@ -124,25 +124,28 @@ namespace KubernetesWorkflow foreach (var ns in namespaces) { - DeleteNamespace(ns); + DeleteNamespace(ns, wait); } } - public void DeleteNamespace() + public void DeleteNamespace(bool wait) { log.Debug(); if (IsNamespaceOnline(K8sNamespace)) { client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0)); + + if (wait) WaitUntilNamespaceDeleted(K8sNamespace); } } - public void DeleteNamespace(string ns) + public void DeleteNamespace(string ns, bool wait) { log.Debug(); if (IsNamespaceOnline(ns)) { client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0)); + if (wait) WaitUntilNamespaceDeleted(ns); } } @@ -871,6 +874,11 @@ namespace KubernetesWorkflow WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated)); } + private void WaitUntilNamespaceDeleted(string @namespace) + { + WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted)); + } + private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index cda6148..3a7326d 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -17,8 +17,8 @@ namespace KubernetesWorkflow void Stop(RunningPod pod, bool waitTillStopped); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); - void DeleteNamespace(); - void DeleteNamespacesStartingWith(string namespacePrefix); + void DeleteNamespace(bool wait); + void DeleteNamespacesStartingWith(string namespacePrefix, bool wait); } public class StartupWorkflow : IStartupWorkflow @@ -122,19 +122,19 @@ namespace KubernetesWorkflow }); } - public void DeleteNamespace() + public void DeleteNamespace(bool wait) { K8s(controller => { - controller.DeleteNamespace(); + controller.DeleteNamespace(wait); }); } - public void DeleteNamespacesStartingWith(string namespacePrefix) + public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait) { K8s(controller => { - controller.DeleteAllNamespacesStartingWith(namespacePrefix); + controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait); }); } diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 12fe874..d394405 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -9,7 +9,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "27-D0-F6-EB-B9-A6-66-41-AA-EA-19-62-07-AF-47-41-25-5E-75-7E-97-35-CC-E1-C0-75-58-17-2D-87-11-75"; + private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 56ceabc..531f82a 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-a518ec6-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index ea7546c..d40fb13 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -108,11 +108,11 @@ namespace CodexPlugin public class CodexSpace { - public int TotalBlocks { get; set; } - public int QuotaMaxBytes { get; set; } - public int QuotaUsedBytes { get; set; } - public int QuotaReservedBytes { get; set; } - public int FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes); + public long TotalBlocks { get; set; } + public long QuotaMaxBytes { get; set; } + public long QuotaUsedBytes { get; set; } + public long QuotaReservedBytes { get; set; } + public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes); public override string ToString() { diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index 6887de9..94450bf 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -289,7 +289,7 @@ components: description: "Root hash of the content" originalBytes: type: integer - format: uint64 + format: int64 description: "Length of original content in bytes" blockSize: type: integer @@ -304,18 +304,18 @@ components: totalBlocks: description: "Number of blocks stored by the node" type: integer - format: uint64 + format: int64 quotaMaxBytes: type: integer - format: uint64 + format: int64 description: "Maximum storage space used by the node" quotaUsedBytes: type: integer - format: uint64 + format: int64 description: "Amount of storage space currently in use" quotaReservedBytes: type: integer - format: uint64 + format: int64 description: "Amount of storage space reserved" servers: diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs index b860c86..fde65aa 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs @@ -148,7 +148,7 @@ namespace ContinuousTests log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log); - entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace); + entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true); } private void PerformCleanup(ILog log) @@ -157,7 +157,7 @@ namespace ContinuousTests log.Log("Cleaning up test namespace..."); var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log); - entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true); + entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true); log.Log("Cleanup finished."); } } diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index 31f1f2e..e58facc 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -64,7 +64,7 @@ namespace ContinuousTests } finally { - entryPoint.Tools.CreateWorkflow().DeleteNamespace(); + entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false); } } diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 38bc502..95d0466 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -54,7 +54,8 @@ namespace ContinuousTests entryPoint.Decommission( deleteKubernetesResources: false, // This would delete the continuous test net. - deleteTrackedFiles: true + deleteTrackedFiles: true, + waitTillDone: false ); runFinishedHandle.Set(); } diff --git a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs index e75d189..1b759ac 100644 --- a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs @@ -18,6 +18,7 @@ public class ScalabilityTests : CodexDistTest [Combinatorial] [UseLongTimeouts] [DontDownloadLogs] + [WaitForCleanup] public void ShouldMaintainFileInNetwork( [Values(10, 40)] int numberOfNodes, // TODO: include 80 and 100 [Values(100, 1000, 5000, 10000)] int fileSizeInMb @@ -64,6 +65,7 @@ public class ScalabilityTests : CodexDistTest [Combinatorial] [UseLongTimeouts] [DontDownloadLogs] + [WaitForCleanup] public void EveryoneGetsAFile( [Values(10, 40, 80, 100)] int numberOfNodes, [Values(100, 1000, 5000, 10000)] int fileSizeInMb diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index ed99fe9..a2edfec 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -52,7 +52,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix); + globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true); }); } catch (Exception ex) @@ -72,7 +72,8 @@ namespace DistTestCore globalEntryPoint.Decommission( // There shouldn't be any of either, but clean everything up regardless. deleteKubernetesResources: true, - deleteTrackedFiles: true + deleteTrackedFiles: true, + waitTillDone: true ); } @@ -185,7 +186,13 @@ namespace DistTestCore lock (lifecycleLock) { var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); - var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId); + var lifecycle = new TestLifecycle( + fixtureLog.CreateTestLog(), + configuration, + GetTimeSet(), + testNamespace, + deployId, + ShouldWaitForCleanup()); lifecycles.Add(testName, lifecycle); LifecycleStart(lifecycle); } @@ -235,6 +242,11 @@ namespace DistTestCore return new DefaultTimeSet(); } + private bool ShouldWaitForCleanup() + { + return CurrentTestMethodHasAttribute(); + } + private bool ShouldUseLongTimeouts() { return CurrentTestMethodHasAttribute(); diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 542ca27..4191cd8 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -16,7 +16,7 @@ namespace DistTestCore private readonly List runningContainers = new(); private readonly string deployId; - public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId) + public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup) { Log = log; Configuration = configuration; @@ -27,7 +27,7 @@ namespace DistTestCore metadata = entryPoint.GetPluginMetadata(); CoreInterface = entryPoint.CreateInterface(); this.deployId = deployId; - + WaitForCleanup = waitForCleanup; log.WriteLogTag(); } @@ -35,13 +35,15 @@ namespace DistTestCore public TestLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } + public bool WaitForCleanup { get; } public CoreInterface CoreInterface { get; } public void DeleteAllResources() { entryPoint.Decommission( deleteKubernetesResources: true, - deleteTrackedFiles: true + deleteTrackedFiles: true, + waitTillDone: WaitForCleanup ); } diff --git a/Tests/DistTestCore/WaitForCleanupAttribute.cs b/Tests/DistTestCore/WaitForCleanupAttribute.cs new file mode 100644 index 0000000..928e3c2 --- /dev/null +++ b/Tests/DistTestCore/WaitForCleanupAttribute.cs @@ -0,0 +1,15 @@ +using NUnit.Framework; + +namespace DistTestCore +{ + /// + /// By default, test system does not wait until all resources are destroyed before starting the + /// next test. This saves a lot of time but it's not always what you want. + /// If you want to be sure the resources of your test are destroyed before the next test starts, + /// add this attribute to your test method. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class WaitForCleanupAttribute : PropertyAttribute + { + } +}