status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-01-11 09:06:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adds WaitForCleanup test attribute to allow tests to wait for resources to be cleaned up

Browse Source
This commit is contained in:
benbierens 2024-06-06 15:09:52 +02:00
parent 38c2d1749a
commit 3a61fc89c6
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
16 changed files with 89 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Framework/Core/EntryPoint.cs
View File

@ -38,10 +38,14 @@ namespace Core
return new CoreInterface(this);
}
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
/// <summary>
/// Deletes kubernetes and tracked file resources.
/// when `waitTillDone` is true, this function will block until resources are deleted.
/// </summary>
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
}
internal T GetPlugin<T>() where T : IProjectPlugin

4
Framework/Core/PluginManager.cs
View File

@ -34,12 +34,12 @@
return metadata;
}
internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
foreach (var pair in pairs)
{
pair.Plugin.Decommission();
pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
}
}

11
Framework/Core/PluginTools.cs
View File

@ -7,7 +7,12 @@ namespace Core
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
{
ITimeSet TimeSet { get; }
void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
/// <summary>
/// Deletes kubernetes and tracked file resources.
/// when `waitTillDone` is true, this function will block until resources are deleted.
/// </summary>
void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
}
public interface IWorkflowTool
@ -73,9 +78,9 @@ namespace Core
return workflowCreator.CreateWorkflow(namespaceOverride);
}
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
{
if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
if (deleteTrackedFiles) fileManager.DeleteAllFiles();
}

16
Framework/KubernetesWorkflow/K8sController.cs
View File

@ -115,7 +115,7 @@ namespace KubernetesWorkflow
});
}
public void DeleteAllNamespacesStartingWith(string prefix)
public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
{
log.Debug();
@ -124,25 +124,28 @@ namespace KubernetesWorkflow
foreach (var ns in namespaces)
{
DeleteNamespace(ns);
DeleteNamespace(ns, wait);
}
}
public void DeleteNamespace()
public void DeleteNamespace(bool wait)
{
log.Debug();
if (IsNamespaceOnline(K8sNamespace))
{
client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
}
}
public void DeleteNamespace(string ns)
public void DeleteNamespace(string ns, bool wait)
{
log.Debug();
if (IsNamespaceOnline(ns))
{
client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
if (wait) WaitUntilNamespaceDeleted(ns);
}
}
@ -871,6 +874,11 @@ namespace KubernetesWorkflow
WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
}
private void WaitUntilNamespaceDeleted(string @namespace)
{
WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
}
private void WaitUntilDeploymentOnline(string deploymentName)
{
WaitUntil(() =>

12
Framework/KubernetesWorkflow/StartupWorkflow.cs
View File

@ -17,8 +17,8 @@ namespace KubernetesWorkflow
void Stop(RunningPod pod, bool waitTillStopped);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteNamespace();
void DeleteNamespacesStartingWith(string namespacePrefix);
void DeleteNamespace(bool wait);
void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
}
public class StartupWorkflow : IStartupWorkflow
@ -122,19 +122,19 @@ namespace KubernetesWorkflow
});
}
public void DeleteNamespace()
public void DeleteNamespace(bool wait)
{
K8s(controller =>
{
controller.DeleteNamespace();
controller.DeleteNamespace(wait);
});
}
public void DeleteNamespacesStartingWith(string namespacePrefix)
public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
{
K8s(controller =>
{
controller.DeleteAllNamespacesStartingWith(namespacePrefix);
controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
});
}

2
ProjectPlugins/CodexPlugin/ApiChecker.cs
View File

@ -9,7 +9,7 @@ namespace CodexPlugin
public class ApiChecker
{
// <INSERT-OPENAPI-YAML-HASH>
private const string OpenApiYamlHash = "27-D0-F6-EB-B9-A6-66-41-AA-EA-19-62-07-AF-47-41-25-5E-75-7E-97-35-CC-E1-C0-75-58-17-2D-87-11-75";
private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";

2
ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
View File

@ -7,7 +7,7 @@ namespace CodexPlugin
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-a518ec6-dist-tests";
private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
public const string ApiPortTag = "codex_api_port";
public const string ListenPortTag = "codex_listen_port";

10
ProjectPlugins/CodexPlugin/CodexTypes.cs
View File

@ -108,11 +108,11 @@ namespace CodexPlugin
public class CodexSpace
{
public int TotalBlocks { get; set; }
public int QuotaMaxBytes { get; set; }
public int QuotaUsedBytes { get; set; }
public int QuotaReservedBytes { get; set; }
public int FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
public long TotalBlocks { get; set; }
public long QuotaMaxBytes { get; set; }
public long QuotaUsedBytes { get; set; }
public long QuotaReservedBytes { get; set; }
public long FreeBytes => QuotaMaxBytes - (QuotaUsedBytes + QuotaReservedBytes);
public override string ToString()
{

10
ProjectPlugins/CodexPlugin/openapi.yaml
View File

@ -289,7 +289,7 @@ components:
description: "Root hash of the content"
originalBytes:
type: integer
format: uint64
format: int64
description: "Length of original content in bytes"
blockSize:
type: integer
@ -304,18 +304,18 @@ components:
totalBlocks:
description: "Number of blocks stored by the node"
type: integer
format: uint64
format: int64
quotaMaxBytes:
type: integer
format: uint64
format: int64
description: "Maximum storage space used by the node"
quotaUsedBytes:
type: integer
format: uint64
format: int64
description: "Amount of storage space currently in use"
quotaReservedBytes:
type: integer
format: uint64
format: int64
description: "Amount of storage space reserved"
servers:

4
Tests/CodexContinuousTests/ContinuousTestRunner.cs
View File

@ -148,7 +148,7 @@ namespace ContinuousTests
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, test.CustomK8sNamespace, log);
entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace);
entryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(test.CustomK8sNamespace, wait: true);
}
private void PerformCleanup(ILog log)
@ -157,7 +157,7 @@ namespace ContinuousTests
log.Log("Cleaning up test namespace...");
var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log);
entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true);
entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true);
log.Log("Cleanup finished.");
}
}

2
Tests/CodexContinuousTests/NodeRunner.cs
View File

@ -64,7 +64,7 @@ namespace ContinuousTests
}
finally
{
entryPoint.Tools.CreateWorkflow().DeleteNamespace();
entryPoint.Tools.CreateWorkflow().DeleteNamespace(wait: false);
}
}

3
Tests/CodexContinuousTests/SingleTestRun.cs
View File

@ -54,7 +54,8 @@ namespace ContinuousTests
entryPoint.Decommission(
deleteKubernetesResources: false, // This would delete the continuous test net.
deleteTrackedFiles: true
deleteTrackedFiles: true,
waitTillDone: false
);
runFinishedHandle.Set();
}

2
Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs
View File

@ -18,6 +18,7 @@ public class ScalabilityTests : CodexDistTest
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
[WaitForCleanup]
public void ShouldMaintainFileInNetwork(
[Values(10, 40)] int numberOfNodes, // TODO: include 80 and 100
[Values(100, 1000, 5000, 10000)] int fileSizeInMb
@ -64,6 +65,7 @@ public class ScalabilityTests : CodexDistTest
[Combinatorial]
[UseLongTimeouts]
[DontDownloadLogs]
[WaitForCleanup]
public void EveryoneGetsAFile(
[Values(10, 40, 80, 100)] int numberOfNodes,
[Values(100, 1000, 5000, 10000)] int fileSizeInMb

18
Tests/DistTestCore/DistTest.cs
View File

@ -52,7 +52,7 @@ namespace DistTestCore
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true);
});
}
catch (Exception ex)
@ -72,7 +72,8 @@ namespace DistTestCore
globalEntryPoint.Decommission(
// There shouldn't be any of either, but clean everything up regardless.
deleteKubernetesResources: true,
deleteTrackedFiles: true
deleteTrackedFiles: true,
waitTillDone: true
);
}
@ -185,7 +186,13 @@ namespace DistTestCore
lock (lifecycleLock)
{
var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
var lifecycle = new TestLifecycle(
fixtureLog.CreateTestLog(),
configuration,
GetTimeSet(),
testNamespace,
deployId,
ShouldWaitForCleanup());
lifecycles.Add(testName, lifecycle);
LifecycleStart(lifecycle);
}
@ -235,6 +242,11 @@ namespace DistTestCore
return new DefaultTimeSet();
}
private bool ShouldWaitForCleanup()
{
return CurrentTestMethodHasAttribute<WaitForCleanupAttribute>();
}
private bool ShouldUseLongTimeouts()
{
return CurrentTestMethodHasAttribute<UseLongTimeoutsAttribute>();

8
Tests/DistTestCore/TestLifecycle.cs
View File

@ -16,7 +16,7 @@ namespace DistTestCore
private readonly List<RunningPod> runningContainers = new();
private readonly string deployId;
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId, bool waitForCleanup)
{
Log = log;
Configuration = configuration;
@ -27,7 +27,7 @@ namespace DistTestCore
metadata = entryPoint.GetPluginMetadata();
CoreInterface = entryPoint.CreateInterface();
this.deployId = deployId;
WaitForCleanup = waitForCleanup;
log.WriteLogTag();
}
@ -35,13 +35,15 @@ namespace DistTestCore
public TestLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public bool WaitForCleanup { get; }
public CoreInterface CoreInterface { get; }
public void DeleteAllResources()
{
entryPoint.Decommission(
deleteKubernetesResources: true,
deleteTrackedFiles: true
deleteTrackedFiles: true,
waitTillDone: WaitForCleanup
);
}

15
Tests/DistTestCore/WaitForCleanupAttribute.cs Normal file
View File

@ -0,0 +1,15 @@
using NUnit.Framework;
namespace DistTestCore
{
/// <summary>
/// By default, test system does not wait until all resources are destroyed before starting the
/// next test. This saves a lot of time but it's not always what you want.
/// If you want to be sure the resources of your test are destroyed before the next test starts,
/// add this attribute to your test method.
/// </summary>
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class WaitForCleanupAttribute : PropertyAttribute
{
}
}