diff --git a/Framework/Core/EntryPoint.cs b/Framework/Core/EntryPoint.cs
index 7977eb3..0db40f3 100644
--- a/Framework/Core/EntryPoint.cs
+++ b/Framework/Core/EntryPoint.cs
@@ -38,10 +38,14 @@ namespace Core
return new CoreInterface(this);
}
- public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+ ///
+ /// Deletes kubernetes and tracked file resources.
+ /// when `waitTillDone` is true, this function will block until resources are deleted.
+ ///
+ public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
- manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
- Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+ manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+ Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
}
internal T GetPlugin() where T : IProjectPlugin
diff --git a/Framework/Core/PluginManager.cs b/Framework/Core/PluginManager.cs
index e2b2a5c..27b08fe 100644
--- a/Framework/Core/PluginManager.cs
+++ b/Framework/Core/PluginManager.cs
@@ -34,12 +34,12 @@
return metadata;
}
- internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+ internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
foreach (var pair in pairs)
{
pair.Plugin.Decommission();
- pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+ pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
}
}
diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs
index 78f7814..aa8e0a6 100644
--- a/Framework/Core/PluginTools.cs
+++ b/Framework/Core/PluginTools.cs
@@ -7,7 +7,12 @@ namespace Core
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
{
ITimeSet TimeSet { get; }
- void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+
+ ///
+ /// Deletes kubernetes and tracked file resources.
+ /// when `waitTillDone` is true, this function will block until resources are deleted.
+ ///
+ void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
}
public interface IWorkflowTool
@@ -73,9 +78,9 @@ namespace Core
return workflowCreator.CreateWorkflow(namespaceOverride);
}
- public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+ public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
- if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+ if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
if (deleteTrackedFiles) fileManager.DeleteAllFiles();
}
diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs
index 0f50d87..30de0c9 100644
--- a/Framework/KubernetesWorkflow/K8sController.cs
+++ b/Framework/KubernetesWorkflow/K8sController.cs
@@ -115,7 +115,7 @@ namespace KubernetesWorkflow
});
}
- public void DeleteAllNamespacesStartingWith(string prefix)
+ public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
{
log.Debug();
@@ -124,25 +124,28 @@ namespace KubernetesWorkflow
foreach (var ns in namespaces)
{
- DeleteNamespace(ns);
+ DeleteNamespace(ns, wait);
}
}
- public void DeleteNamespace()
+ public void DeleteNamespace(bool wait)
{
log.Debug();
if (IsNamespaceOnline(K8sNamespace))
{
client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+ if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
}
}
- public void DeleteNamespace(string ns)
+ public void DeleteNamespace(string ns, bool wait)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+ if (wait) WaitUntilNamespaceDeleted(ns);
}
}
@@ -871,6 +874,11 @@ namespace KubernetesWorkflow
WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
}
+ private void WaitUntilNamespaceDeleted(string @namespace)
+ {
+ WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+ }
+
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>
diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs
index cda6148..3a7326d 100644
--- a/Framework/KubernetesWorkflow/StartupWorkflow.cs
+++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs
@@ -17,8 +17,8 @@ namespace KubernetesWorkflow
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
- void DeleteNamespace();
- void DeleteNamespacesStartingWith(string namespacePrefix);
+ void DeleteNamespace(bool wait);
+ void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
}
public class StartupWorkflow : IStartupWorkflow
@@ -122,19 +122,19 @@ namespace KubernetesWorkflow
});
}
- public void DeleteNamespace()
+ public void DeleteNamespace(bool wait)
{
K8s(controller =>
{
- controller.DeleteNamespace();
+ controller.DeleteNamespace(wait);
});
}
- public void DeleteNamespacesStartingWith(string namespacePrefix)
+ public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
{
K8s(controller =>
{
- controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+ controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
});
}
diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs
index 12fe874..d394405 100644
--- a/ProjectPlugins/CodexPlugin/ApiChecker.cs
+++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs
@@ -9,7 +9,7 @@ namespace CodexPlugin
public class ApiChecker
{
//
- private const string OpenApiYamlHash = "27-D0-F6-EB-B9-A6-66-41-AA-EA-19-62-07-AF-47-41-25-5E-75-7E-97-35-CC-E1-C0-75-58-17-2D-87-11-75";
+ private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
index 56ceabc..531f82a 100644
--- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
+++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
@@ -7,7 +7,7 @@ namespace CodexPlugin
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
- private const string DefaultDockerImage = "codexstorage/nim-codex:sha-a518ec6-dist-tests";
+ private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
public const string ApiPortTag = "codex_api_port";
public const string ListenPortTag = "codex_listen_port";
diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs
index ea7546c..d40fb13 100644
--- a/ProjectPlugins/CodexPlugin/CodexTypes.cs
+++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs
@@ -108,11 +108,11 @@ namespace CodexPlugin
public class CodexSpace
{
- public int TotalBlocks { get; set; }
- public int QuotaMaxBytes { get; set; }
- public int QuotaUsedBytes { get; set; }
- public int QuotaReservedBytes { get; set; }
- public int FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
+ public long TotalBlocks { get; set; }
+ public long QuotaMaxBytes { get; set; }
+ public long QuotaUsedBytes { get; set; }
+ public long QuotaReservedBytes { get; set; }
+ public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
public override string ToString()
{
diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml
index 6887de9..94450bf 100644
--- a/ProjectPlugins/CodexPlugin/openapi.yaml
+++ b/ProjectPlugins/CodexPlugin/openapi.yaml
@@ -289,7 +289,7 @@ components:
description: "Root hash of the content"
originalBytes:
type: integer
- format: uint64
+ format: int64
description: "Length of original content in bytes"
blockSize:
type: integer
@@ -304,18 +304,18 @@ components:
totalBlocks:
description: "Number of blocks stored by the node"
type: integer
- format: uint64
+ format: int64
quotaMaxBytes:
type: integer
- format: uint64
+ format: int64
description: "Maximum storage space used by the node"
quotaUsedBytes:
type: integer
- format: uint64
+ format: int64
description: "Amount of storage space currently in use"
quotaReservedBytes:
type: integer
- format: uint64
+ format: int64
description: "Amount of storage space reserved"
servers:
diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs
index b860c86..fde65aa 100644
--- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs
+++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs
@@ -148,7 +148,7 @@ namespace ContinuousTests
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
- entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
+ entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
}
private void PerformCleanup(ILog log)
@@ -157,7 +157,7 @@ namespace ContinuousTests
log.Log("Cleaning up test namespace...");
var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
- entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
+ entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
log.Log("Cleanup finished.");
}
}
diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs
index 31f1f2e..e58facc 100644
--- a/Tests/CodexContinuousTests/NodeRunner.cs
+++ b/Tests/CodexContinuousTests/NodeRunner.cs
@@ -64,7 +64,7 @@ namespace ContinuousTests
}
finally
{
- entryPoint.Tools.CreateWorkflow().DeleteNamespace();
+ entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false);
}
}
diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs
index 38bc502..95d0466 100644
--- a/Tests/CodexContinuousTests/SingleTestRun.cs
+++ b/Tests/CodexContinuousTests/SingleTestRun.cs
@@ -54,7 +54,8 @@ namespace ContinuousTests
entryPoint.Decommission(
deleteKubernetesResources: false, // This would delete the continuous test net.
- deleteTrackedFiles: true
+ deleteTrackedFiles: true,
+ waitTillDone: false
);
runFinishedHandle.Set();
}
diff --git a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs
index e75d189..1b759ac 100644
--- a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs
+++ b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs
@@ -18,6 +18,7 @@ public class ScalabilityTests : CodexDistTest
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
+ [WaitForCleanup]
public void ShouldMaintainFileInNetwork(
[Values(10, 40)] int numberOfNodes, // TODO: include 80 and 100
[Values(100, 1000, 5000, 10000)] int fileSizeInMb
@@ -64,6 +65,7 @@ public class ScalabilityTests : CodexDistTest
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
+ [WaitForCleanup]
public void EveryoneGetsAFile(
[Values(10, 40, 80, 100)] int numberOfNodes,
[Values(100, 1000, 5000, 10000)] int fileSizeInMb
diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs
index ed99fe9..a2edfec 100644
--- a/Tests/DistTestCore/DistTest.cs
+++ b/Tests/DistTestCore/DistTest.cs
@@ -52,7 +52,7 @@ namespace DistTestCore
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
- globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
+ globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
});
}
catch (Exception ex)
@@ -72,7 +72,8 @@ namespace DistTestCore
globalEntryPoint.Decommission(
// There shouldn't be any of either, but clean everything up regardless.
deleteKubernetesResources: true,
- deleteTrackedFiles: true
+ deleteTrackedFiles: true,
+ waitTillDone: true
);
}
@@ -185,7 +186,13 @@ namespace DistTestCore
lock (lifecycleLock)
{
var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
- var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
+ var lifecycle = new TestLifecycle(
+ fixtureLog.CreateTestLog(),
+ configuration,
+ GetTimeSet(),
+ testNamespace,
+ deployId,
+ ShouldWaitForCleanup());
lifecycles.Add(testName, lifecycle);
LifecycleStart(lifecycle);
}
@@ -235,6 +242,11 @@ namespace DistTestCore
return new DefaultTimeSet();
}
+ private bool ShouldWaitForCleanup()
+ {
+ return CurrentTestMethodHasAttribute();
+ }
+
private bool ShouldUseLongTimeouts()
{
return CurrentTestMethodHasAttribute();
diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs
index 542ca27..4191cd8 100644
--- a/Tests/DistTestCore/TestLifecycle.cs
+++ b/Tests/DistTestCore/TestLifecycle.cs
@@ -16,7 +16,7 @@ namespace DistTestCore
private readonly List runningContainers = new();
private readonly string deployId;
- public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
+ public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
{
Log = log;
Configuration = configuration;
@@ -27,7 +27,7 @@ namespace DistTestCore
metadata = entryPoint.GetPluginMetadata();
CoreInterface = entryPoint.CreateInterface();
this.deployId = deployId;
-
+ WaitForCleanup = waitForCleanup;
log.WriteLogTag();
}
@@ -35,13 +35,15 @@ namespace DistTestCore
public TestLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
+ public bool WaitForCleanup { get; }
public CoreInterface CoreInterface { get; }
public void DeleteAllResources()
{
entryPoint.Decommission(
deleteKubernetesResources: true,
- deleteTrackedFiles: true
+ deleteTrackedFiles: true,
+ waitTillDone: WaitForCleanup
);
}
diff --git a/Tests/DistTestCore/WaitForCleanupAttribute.cs b/Tests/DistTestCore/WaitForCleanupAttribute.cs
new file mode 100644
index 0000000..928e3c2
--- /dev/null
+++ b/Tests/DistTestCore/WaitForCleanupAttribute.cs
@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+ ///
+ /// By default, test system does not wait until all resources are destroyed before starting the
+ /// next test. This saves a lot of time but it's not always what you want.
+ /// If you want to be sure the resources of your test are destroyed before the next test starts,
+ /// add this attribute to your test method.
+ ///
+ [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+ public class WaitForCleanupAttribute : PropertyAttribute
+ {
+ }
+}