From 7db9360ba411da2b24a56d0c3957d24d340b333e Mon Sep 17 00:00:00 2001 From: gmega Date: Sat, 13 Apr 2024 16:27:47 +0300 Subject: [PATCH 01/22] add network/file scalability test --- .../ScalabilityTests/ScalabilityTests.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs new file mode 100644 index 00000000..c597d49e --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -0,0 +1,43 @@ +using CodexPlugin; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests; + +[TestFixture] +public class ScalabilityTests : CodexDistTest +{ + private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests"; + private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests"; + + [Test] + [Combinatorial] + public void ShouldMaintainFileInNetwork( + [Values(10, 20, 40, 80, 100)] int numberOfNodes, + [Values(100, 1000, 5000, 10000)] int fileSizeInMb, + [Values(true, false)] bool usePatchedImage + ) + { + CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; + + var bootstrap = AddCodex(); + var nodes = AddCodex(numberOfNodes - 1, + s => s.WithBootstrapNode(bootstrap)).ToList(); + + var uploader = nodes.PickOneRandom(); + var downloader = nodes.PickOneRandom(); + + var testFile = GenerateTestFile(fileSizeInMb.MB()); + var contentId = uploader.UploadFile(testFile); + var downloadedFile = downloader.DownloadContent(contentId); + + downloadedFile!.AssertIsEqual(testFile); + + uploader.Stop(true); + + var otherDownloader = nodes.PickOneRandom(); + downloadedFile = otherDownloader.DownloadContent(contentId); + + downloadedFile!.AssertIsEqual(testFile); + } +} \ No newline at end of file From 80261959e74139abcc988b866a58bca5ce3befef Mon Sep 17 00:00:00 2001 From: gmega Date: Sat, 13 Apr 2024 16:36:16 +0300 Subject: [PATCH 02/22] set log level to info --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index c597d49e..b767f39b 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -19,10 +19,10 @@ public class ScalabilityTests : CodexDistTest ) { CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; - + var bootstrap = AddCodex(); var nodes = AddCodex(numberOfNodes - 1, - s => s.WithBootstrapNode(bootstrap)).ToList(); + s => s.WithBootstrapNode(bootstrap).WithLogLevel(CodexLogLevel.Info)).ToList(); var uploader = nodes.PickOneRandom(); var downloader = nodes.PickOneRandom(); From e3b16fd7420a890930827fe421aea86f30a730e0 Mon Sep 17 00:00:00 2001 From: gmega Date: Sat, 13 Apr 2024 17:09:17 +0300 Subject: [PATCH 03/22] add ability to stop single containers --- Framework/KubernetesWorkflow/K8sController.cs | 2 +- Framework/KubernetesWorkflow/K8sHooks.cs | 8 +++---- .../KubernetesWorkflow/StartupWorkflow.cs | 20 ++++++++-------- .../Types/FutureContainers.cs | 12 +++++----- .../Types/RunningContainer.cs | 2 +- .../{RunningContainers.cs => RunningPod.cs} | 13 ++++------ .../CodexDiscordBotPlugin.cs | 8 +++---- .../CoreInterfaceExtensions.cs | 4 ++-- ProjectPlugins/CodexPlugin/ApiChecker.cs | 2 +- ProjectPlugins/CodexPlugin/CodexAccess.cs | 6 ++--- ProjectPlugins/CodexPlugin/CodexDeployment.cs | 14 +++++------ ProjectPlugins/CodexPlugin/CodexNode.cs | 20 +++++++++------- .../CodexPlugin/CodexNodeFactory.cs | 2 +- ProjectPlugins/CodexPlugin/CodexNodeGroup.cs | 17 +++++++++---- ProjectPlugins/CodexPlugin/CodexPlugin.cs | 4 ++-- ProjectPlugins/CodexPlugin/CodexStarter.cs | 24 ++++++++++++------- .../CodexPlugin/CoreInterfaceExtensions.cs | 4 ++-- ProjectPlugins/GethPlugin/GethDeployment.cs | 8 +++---- .../MetricsPlugin/CoreInterfaceExtensions.cs | 12 +++++----- ProjectPlugins/MetricsPlugin/MetricsPlugin.cs | 8 +++---- .../MetricsPlugin/PrometheusStarter.cs | 6 ++--- .../ElasticSearchLogDownloader.cs | 4 ++-- Tests/CodexContinuousTests/SingleTestRun.cs | 10 ++++---- Tests/CodexContinuousTests/StartupChecker.cs | 8 +++---- .../BasicTests/ContinuousSubstitute.cs | 2 +- Tests/DistTestCore/TestLifecycle.cs | 6 ++--- Tools/CodexNetDeployer/Deployer.cs | 6 ++--- Tools/CodexNetDeployer/K8sHook.cs | 4 ++-- 28 files changed, 125 insertions(+), 111 deletions(-) rename Framework/KubernetesWorkflow/Types/{RunningContainers.cs => RunningPod.cs} (61%) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 80b15d16..a17ad823 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -705,7 +705,7 @@ namespace KubernetesWorkflow private string GetPodName(RunningContainer container) { - return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name; + return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name; } private V1Pod GetPodForDeployment(RunningDeployment deployment) diff --git a/Framework/KubernetesWorkflow/K8sHooks.cs b/Framework/KubernetesWorkflow/K8sHooks.cs index 74bb93b4..4dad7401 100644 --- a/Framework/KubernetesWorkflow/K8sHooks.cs +++ b/Framework/KubernetesWorkflow/K8sHooks.cs @@ -5,18 +5,18 @@ namespace KubernetesWorkflow { public interface IK8sHooks { - void OnContainersStarted(RunningContainers runningContainers); - void OnContainersStopped(RunningContainers runningContainers); + void OnContainersStarted(RunningPod runningPod); + void OnContainersStopped(RunningPod runningPod); void OnContainerRecipeCreated(ContainerRecipe recipe); } public class DoNothingK8sHooks : IK8sHooks { - public void OnContainersStarted(RunningContainers runningContainers) + public void OnContainersStarted(RunningPod runningPod) { } - public void OnContainersStopped(RunningContainers runningContainers) + public void OnContainersStopped(RunningPod runningPod) { } diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index b8f7d779..cda61489 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -12,9 +12,9 @@ namespace KubernetesWorkflow FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); PodInfo GetPodInfo(RunningContainer container); - PodInfo GetPodInfo(RunningContainers containers); + PodInfo GetPodInfo(RunningPod pod); CrashWatcher CreateCrashWatcher(RunningContainer container); - void Stop(RunningContainers containers, bool waitTillStopped); + void Stop(RunningPod pod, bool waitTillStopped); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); void DeleteNamespace(); @@ -60,7 +60,7 @@ namespace KubernetesWorkflow var startResult = controller.BringOnline(recipes, location); var containers = CreateContainers(startResult, recipes, startupConfig); - var rc = new RunningContainers(startupConfig, startResult, containers); + var rc = new RunningPod(startupConfig, startResult, containers); cluster.Configuration.Hooks.OnContainersStarted(rc); if (startResult.ExternalService != null) @@ -71,7 +71,7 @@ namespace KubernetesWorkflow }); } - public void WaitUntilOnline(RunningContainers rc) + public void WaitUntilOnline(RunningPod rc) { K8s(controller => { @@ -84,12 +84,12 @@ namespace KubernetesWorkflow public PodInfo GetPodInfo(RunningContainer container) { - return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment)); + return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment)); } - public PodInfo GetPodInfo(RunningContainers containers) + public PodInfo GetPodInfo(RunningPod pod) { - return K8s(c => c.GetPodInfo(containers.StartResult.Deployment)); + return K8s(c => c.GetPodInfo(pod.StartResult.Deployment)); } public CrashWatcher CreateCrashWatcher(RunningContainer container) @@ -97,12 +97,12 @@ namespace KubernetesWorkflow return K8s(c => c.CreateCrashWatcher(container)); } - public void Stop(RunningContainers runningContainers, bool waitTillStopped) + public void Stop(RunningPod runningPod, bool waitTillStopped) { K8s(controller => { - controller.Stop(runningContainers.StartResult, waitTillStopped); - cluster.Configuration.Hooks.OnContainersStopped(runningContainers); + controller.Stop(runningPod.StartResult, waitTillStopped); + cluster.Configuration.Hooks.OnContainersStopped(runningPod); }); } diff --git a/Framework/KubernetesWorkflow/Types/FutureContainers.cs b/Framework/KubernetesWorkflow/Types/FutureContainers.cs index 262eac44..296be534 100644 --- a/Framework/KubernetesWorkflow/Types/FutureContainers.cs +++ b/Framework/KubernetesWorkflow/Types/FutureContainers.cs @@ -2,19 +2,19 @@ { public class FutureContainers { - private readonly RunningContainers runningContainers; + private readonly RunningPod runningPod; private readonly StartupWorkflow workflow; - public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow) + public FutureContainers(RunningPod runningPod, StartupWorkflow workflow) { - this.runningContainers = runningContainers; + this.runningPod = runningPod; this.workflow = workflow; } - public RunningContainers WaitForOnline() + public RunningPod WaitForOnline() { - workflow.WaitUntilOnline(runningContainers); - return runningContainers; + workflow.WaitUntilOnline(runningPod); + return runningPod; } } } diff --git a/Framework/KubernetesWorkflow/Types/RunningContainer.cs b/Framework/KubernetesWorkflow/Types/RunningContainer.cs index f391e457..b0fbc02e 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainer.cs +++ b/Framework/KubernetesWorkflow/Types/RunningContainer.cs @@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types public ContainerAddress[] Addresses { get; } [JsonIgnore] - public RunningContainers RunningContainers { get; internal set; } = null!; + public RunningPod RunningPod { get; internal set; } = null!; public Address GetAddress(ILog log, string portTag) { diff --git a/Framework/KubernetesWorkflow/Types/RunningContainers.cs b/Framework/KubernetesWorkflow/Types/RunningPod.cs similarity index 61% rename from Framework/KubernetesWorkflow/Types/RunningContainers.cs rename to Framework/KubernetesWorkflow/Types/RunningPod.cs index 9a6e5f3a..7f1a24d7 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/Types/RunningPod.cs @@ -2,15 +2,15 @@ namespace KubernetesWorkflow.Types { - public class RunningContainers + public class RunningPod { - public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) + public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) { StartupConfig = startupConfig; StartResult = startResult; Containers = containers; - foreach (var c in containers) c.RunningContainers = this; + foreach (var c in containers) c.RunningPod = this; } public StartupConfig StartupConfig { get; } @@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types public static class RunningContainersExtensions { - public static RunningContainer[] Containers(this RunningContainers[] runningContainers) - { - return runningContainers.SelectMany(c => c.Containers).ToArray(); - } - - public static string Describe(this RunningContainers[] runningContainers) + public static string Describe(this RunningPod[] runningContainers) { return string.Join(",", runningContainers.Select(c => c.Describe())); } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index fd082cfa..f347805a 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -29,19 +29,19 @@ namespace CodexDiscordBotPlugin { } - public RunningContainers Deploy(DiscordBotStartupConfig config) + public RunningPod Deploy(DiscordBotStartupConfig config) { var workflow = tools.CreateWorkflow(); return StartContainer(workflow, config); } - public RunningContainers DeployRewarder(RewarderBotStartupConfig config) + public RunningPod DeployRewarder(RewarderBotStartupConfig config) { var workflow = tools.CreateWorkflow(); return StartRewarderContainer(workflow, config); } - private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config) + private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; @@ -49,7 +49,7 @@ namespace CodexDiscordBotPlugin return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); } - private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) + private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.Add(config); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs index c17cec15..c91d7118 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs @@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) + public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) { return Plugin(ci).Deploy(config); } - public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config) + public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config) { return Plugin(ci).DeployRewarder(config); } diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 212fa193..5cf3dc96 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -38,7 +38,7 @@ namespace CodexPlugin if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger."); } - public void CheckCompatibility(RunningContainers[] containers) + public void CheckCompatibility(RunningPod[] containers) { if (checkPassed) return; diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 697ff2cd..e35a9f9d 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -13,7 +13,7 @@ namespace CodexPlugin private readonly Mapper mapper = new Mapper(); private bool hasContainerCrashed; - public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher) + public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher) { this.tools = tools; Container = container; @@ -23,7 +23,7 @@ namespace CodexPlugin CrashWatcher.Start(this); } - public RunningContainer Container { get; } + public RunningPod Container { get; } public CrashWatcher CrashWatcher { get; } public DebugInfo GetDebugInfo() @@ -136,7 +136,7 @@ namespace CodexPlugin private Address GetAddress() { - return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); + return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); } private void CheckContainerCrashed(HttpClient client) diff --git a/ProjectPlugins/CodexPlugin/CodexDeployment.cs b/ProjectPlugins/CodexPlugin/CodexDeployment.cs index 9f74dd31..7cbf4506 100644 --- a/ProjectPlugins/CodexPlugin/CodexDeployment.cs +++ b/ProjectPlugins/CodexPlugin/CodexDeployment.cs @@ -7,8 +7,8 @@ namespace CodexPlugin public class CodexDeployment { public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, - CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer, - RunningContainers? discordBotContainer, DeploymentMetadata metadata, + CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer, + RunningPod? discordBotContainer, DeploymentMetadata metadata, String id) { Id = id; @@ -24,20 +24,20 @@ namespace CodexPlugin public CodexInstance[] CodexInstances { get; } public GethDeployment GethDeployment { get; } public CodexContractsDeployment CodexContractsDeployment { get; } - public RunningContainers? PrometheusContainer { get; } - public RunningContainers? DiscordBotContainer { get; } + public RunningPod? PrometheusContainer { get; } + public RunningPod? DiscordBotContainer { get; } public DeploymentMetadata Metadata { get; } } public class CodexInstance { - public CodexInstance(RunningContainers containers, DebugInfo info) + public CodexInstance(RunningPod pod, DebugInfo info) { - Containers = containers; + Pod = pod; Info = info; } - public RunningContainers Containers { get; } + public RunningPod Pod { get; } public DebugInfo Info { get; } } diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index e1e475ce..54299c3c 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -44,7 +44,9 @@ namespace CodexPlugin transferSpeeds = new TransferSpeeds(); } - public RunningContainer Container { get { return CodexAccess.Container; } } + public RunningPod Pod { get { return CodexAccess.Container; } } + + public RunningContainer Container { get { return Pod.Containers.Single(); } } public CodexAccess CodexAccess { get; } public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; } public CodexNodeGroup Group { get; } @@ -56,7 +58,7 @@ namespace CodexPlugin { get { - return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag); + return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag); } } @@ -142,11 +144,13 @@ namespace CodexPlugin public void Stop(bool waitTillStopped) { - if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + - "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + - "available for codex-nodes in groups of 1."); - - Group.BringOffline(waitTillStopped); + Group.Stop(this, waitTillStopped); + CrashWatcher.Stop(); + // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + + // "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + + // "available for codex-nodes in groups of 1."); + // + // Group.BringOffline(waitTillStopped); } public void EnsureOnlineGetVersionResponse() @@ -171,7 +175,7 @@ namespace CodexPlugin // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. var workflow = tools.CreateWorkflow(); - var podInfo = workflow.GetPodInfo(peer.Container); + var podInfo = workflow.GetPodInfo(peer.Pod); return peerInfo.Addrs.Select(a => a .Replace("0.0.0.0", podInfo.Ip)) diff --git a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs index 05b80b28..18483de6 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs @@ -35,7 +35,7 @@ namespace CodexPlugin private EthAddress? GetEthAddress(CodexAccess access) { - var ethAccount = access.Container.Recipe.Additionals.Get(); + var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get(); if (ethAccount == null) return null; return ethAccount.EthAddress; } diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index 1f7ed4eb..47bf35d9 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -15,11 +15,11 @@ namespace CodexPlugin { private readonly CodexStarter starter; - public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory) + public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory) { this.starter = starter; Containers = containers; - Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); + Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); Version = new DebugInfoVersion(); } @@ -39,7 +39,14 @@ namespace CodexPlugin Containers = null!; } - public RunningContainers[] Containers { get; private set; } + public void Stop(CodexNode node, bool waitTillStopped) + { + starter.Stop(node.Pod, waitTillStopped); + Nodes = Nodes.Where(n => n != node).ToArray(); + Containers = Containers.Where(c => c != node.Pod).ToArray(); + } + + public RunningPod[] Containers { get; private set; } public CodexNode[] Nodes { get; private set; } public DebugInfoVersion Version { get; private set; } public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray(); @@ -74,9 +81,9 @@ namespace CodexPlugin Version = first; } - private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory) + private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory) { - var watcher = factory.CreateCrashWatcher(c); + var watcher = factory.CreateCrashWatcher(c.Containers.Single()); var access = new CodexAccess(tools, c, watcher); return factory.CreateOnlineCodexNode(access, this); } diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 9b8586e8..7b722ed8 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -32,13 +32,13 @@ namespace CodexPlugin { } - public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action setup) + public RunningPod[] DeployCodexNodes(int numberOfNodes, Action setup) { var codexSetup = GetSetup(numberOfNodes, setup); return codexStarter.BringOnline(codexSetup); } - public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers) + public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers) { containers = containers.Select(c => SerializeGate.Gate(c)).ToArray(); return codexStarter.WrapCodexContainers(coreInterface, containers); diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 6fcc4bd3..dec13d38 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -19,7 +19,7 @@ namespace CodexPlugin apiChecker = new ApiChecker(pluginTools); } - public RunningContainers[] BringOnline(CodexSetup codexSetup) + public RunningPod[] BringOnline(CodexSetup codexSetup) { LogSeparator(); Log($"Starting {codexSetup.Describe()}..."); @@ -34,14 +34,14 @@ namespace CodexPlugin { var podInfo = GetPodInfo(rc); var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}")); - Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})"); + Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})"); } LogSeparator(); return containers; } - public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers) + public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers) { var codexNodeFactory = new CodexNodeFactory(pluginTools); @@ -65,6 +65,14 @@ namespace CodexPlugin Log("Stopped."); } + public void Stop(RunningPod pod, bool waitTillStopped) + { + Log($"Stopping node..."); + var workflow = pluginTools.CreateWorkflow(); + workflow.Stop(pod, waitTillStopped); + Log("Stopped."); + } + public string GetCodexId() { if (versionResponse != null) return versionResponse.Version; @@ -85,7 +93,7 @@ namespace CodexPlugin return startupConfig; } - private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) + private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) { var futureContainers = new List(); for (var i = 0; i < numberOfNodes; i++) @@ -99,13 +107,13 @@ namespace CodexPlugin .ToArray(); } - private PodInfo GetPodInfo(RunningContainers rc) + private PodInfo GetPodInfo(RunningPod rc) { var workflow = pluginTools.CreateWorkflow(); return workflow.GetPodInfo(rc); } - private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory) + private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory) { var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory); @@ -122,10 +130,10 @@ namespace CodexPlugin return group; } - private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers) + private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers) { Log("Codex nodes failed to start"); - foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container); + foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container); } private void LogSeparator() diff --git a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs index 529b7fd8..5f6065df 100644 --- a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs @@ -5,12 +5,12 @@ namespace CodexPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action setup) + public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action setup) { return Plugin(ci).DeployCodexNodes(number, setup); } - public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers) + public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers) { return Plugin(ci).WrapCodexContainers(ci, containers); } diff --git a/ProjectPlugins/GethPlugin/GethDeployment.cs b/ProjectPlugins/GethPlugin/GethDeployment.cs index 8d7f0dd5..e463ce42 100644 --- a/ProjectPlugins/GethPlugin/GethDeployment.cs +++ b/ProjectPlugins/GethPlugin/GethDeployment.cs @@ -7,9 +7,9 @@ namespace GethPlugin { public class GethDeployment : IHasContainer { - public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey) + public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey) { - Containers = containers; + Pod = pod; DiscoveryPort = discoveryPort; HttpPort = httpPort; WsPort = wsPort; @@ -17,9 +17,9 @@ namespace GethPlugin PubKey = pubKey; } - public RunningContainers Containers { get; } + public RunningPod Pod { get; } [JsonIgnore] - public RunningContainer Container { get { return Containers.Containers.Single(); } } + public RunningContainer Container { get { return Pod.Containers.Single(); } } public Port DiscoveryPort { get; } public Port HttpPort { get; } public Port WsPort { get; } diff --git a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs index 83920825..6d5859da 100644 --- a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs @@ -6,24 +6,24 @@ namespace MetricsPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets) + public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets) { return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray()); } - public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) + public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) { return Plugin(ci).DeployMetricsCollector(scrapeTargets); } - public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget) + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget) { - return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget); + return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget); } - public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget) + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget) { - return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget); + return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget); } public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets) diff --git a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs index b2c2b8ea..bc7d926b 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs @@ -31,15 +31,15 @@ namespace MetricsPlugin { } - public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets) + public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets) { return starter.CollectMetricsFor(scrapeTargets); } - public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target) + public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target) { - runningContainer = SerializeGate.Gate(runningContainer); - return starter.CreateAccessForTarget(runningContainer, target); + runningPod = SerializeGate.Gate(runningPod); + return starter.CreateAccessForTarget(runningPod, target); } public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName) diff --git a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs index f63f97d5..159947a2 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs @@ -16,7 +16,7 @@ namespace MetricsPlugin this.tools = tools; } - public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets) + public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets) { if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty."); @@ -32,9 +32,9 @@ namespace MetricsPlugin return runningContainers; } - public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target) + public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target) { - var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single()); + var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single()); return new MetricsAccess(metricsQuery, target); } diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs index 80203af3..10c909ca 100644 --- a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs +++ b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs @@ -49,8 +49,8 @@ namespace ContinuousTests var start = startUtc.ToString("o"); var end = endUtc.ToString("o"); - var containerName = container.RunningContainers.StartResult.Deployment.Name; - var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace; + var containerName = container.RunningPod.StartResult.Deployment.Name; + var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace; //container_name : codex3-5 - deploymentName as stored in pod // pod_namespace : codex - continuous - nolimits - tests - 1 diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 7c6c5a6f..38bc5021 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -125,8 +125,8 @@ namespace ContinuousTests foreach (var node in nodes) { var container = node.Container; - var deploymentName = container.RunningContainers.StartResult.Deployment.Name; - var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace; + var deploymentName = container.RunningPod.StartResult.Deployment.Name; + var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace; var openingLine = $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}"; elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, @@ -295,13 +295,13 @@ namespace ContinuousTests return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray(); } - private RunningContainers[] SelectRandomContainers() + private RunningPod[] SelectRandomContainers() { var number = handle.Test.RequiredNumberOfNodes; - var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList(); + var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList(); if (number == -1) return containers.ToArray(); - var result = new RunningContainers[number]; + var result = new RunningPod[number]; for (var i = 0; i < number; i++) { result[i] = containers.PickOneRandom(); diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index d3fb456c..57c383c8 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -43,13 +43,13 @@ namespace ContinuousTests var workflow = entryPoint.Tools.CreateWorkflow(); foreach (var instance in deployment.CodexInstances) { - foreach (var container in instance.Containers.Containers) + foreach (var container in instance.Pod.Containers) { var podInfo = workflow.GetPodInfo(container); log.Log($"Codex environment variables for '{container.Name}':"); log.Log( - $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " + - $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}"); + $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " + + $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}"); var codexVars = container.Recipe.EnvVars; foreach (var vars in codexVars) log.Log(vars.ToString()); log.Log(""); @@ -92,7 +92,7 @@ namespace ContinuousTests private void CheckCodexNodes(BaseLog log, Configuration config) { var nodes = entryPoint.CreateInterface() - .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray()); + .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray()); var pass = true; foreach (var n in nodes) { diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 299c4b94..2898be20 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -132,7 +132,7 @@ namespace CodexTests.BasicTests private const string BytesStoredMetric = "codexRepostoreBytesUsed"; - private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc) + private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc) { ScopedTestFiles(() => { diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 50cb5fb2..1421fe59 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -13,7 +13,7 @@ namespace DistTestCore private const string TestsType = "dist-tests"; private readonly EntryPoint entryPoint; private readonly Dictionary metadata; - private readonly List runningContainers = new(); + private readonly List runningContainers = new(); private readonly string deployId; public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId) @@ -65,12 +65,12 @@ namespace DistTestCore return DateTime.UtcNow - TestStart; } - public void OnContainersStarted(RunningContainers rc) + public void OnContainersStarted(RunningPod rc) { runningContainers.Add(rc); } - public void OnContainersStopped(RunningContainers rc) + public void OnContainersStopped(RunningPod rc) { runningContainers.Remove(rc); } diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 440d092a..04171083 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -122,7 +122,7 @@ namespace CodexNetDeployer }); } - private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, + private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, CodexContractsDeployment contractsDeployment) { if (!config.DeployDiscordBot) return null; @@ -155,7 +155,7 @@ namespace CodexNetDeployer return rc; } - private RunningContainers? StartMetricsService(CoreInterface ci, List startResults) + private RunningPod? StartMetricsService(CoreInterface ci, List startResults) { if (!config.MetricsScraper || !startResults.Any()) return null; @@ -180,7 +180,7 @@ namespace CodexNetDeployer private CodexInstance CreateCodexInstance(ICodexNode node) { - return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo()); + return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo()); } private string? GetKubeConfig(string kubeConfigFile) diff --git a/Tools/CodexNetDeployer/K8sHook.cs b/Tools/CodexNetDeployer/K8sHook.cs index b4c0c16a..b8c78b7c 100644 --- a/Tools/CodexNetDeployer/K8sHook.cs +++ b/Tools/CodexNetDeployer/K8sHook.cs @@ -18,11 +18,11 @@ namespace CodexNetDeployer this.metadata = metadata; } - public void OnContainersStarted(RunningContainers rc) + public void OnContainersStarted(RunningPod rc) { } - public void OnContainersStopped(RunningContainers rc) + public void OnContainersStopped(RunningPod rc) { } From 5ffe34bb83e85dcc6df23fbb03080b502c97ab59 Mon Sep 17 00:00:00 2001 From: gmega Date: Sat, 13 Apr 2024 17:20:23 +0300 Subject: [PATCH 04/22] stop crash watcher before stopping pod --- ProjectPlugins/CodexPlugin/CodexNode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 54299c3c..38b8d418 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -144,8 +144,8 @@ namespace CodexPlugin public void Stop(bool waitTillStopped) { - Group.Stop(this, waitTillStopped); CrashWatcher.Stop(); + Group.Stop(this, waitTillStopped); // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + // "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + // "available for codex-nodes in groups of 1."); From a2e4869403d38ae8195b8c29df85b320ec5cad7b Mon Sep 17 00:00:00 2001 From: gmega Date: Sat, 13 Apr 2024 17:33:21 +0300 Subject: [PATCH 05/22] use long timeouts --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index b767f39b..b66345a3 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -1,4 +1,5 @@ using CodexPlugin; +using DistTestCore; using NUnit.Framework; using Utils; @@ -12,6 +13,7 @@ public class ScalabilityTests : CodexDistTest [Test] [Combinatorial] + [UseLongTimeouts] public void ShouldMaintainFileInNetwork( [Values(10, 20, 40, 80, 100)] int numberOfNodes, [Values(100, 1000, 5000, 10000)] int fileSizeInMb, From d847c4f3ec757d70c94fc1fadbfe2111a84350e7 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 08:56:22 +0200 Subject: [PATCH 06/22] Adds names to kube wait functions --- Framework/KubernetesWorkflow/K8sController.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index a17ad823..d81b87c4 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -868,7 +868,7 @@ namespace KubernetesWorkflow private void WaitUntilNamespaceCreated() { - WaitUntil(() => IsNamespaceOnline(K8sNamespace)); + WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated)); } private void WaitUntilDeploymentOnline(string deploymentName) @@ -877,7 +877,7 @@ namespace KubernetesWorkflow { var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; - }); + }, nameof(WaitUntilDeploymentOnline)); } private void WaitUntilDeploymentOffline(string deploymentName) @@ -887,7 +887,7 @@ namespace KubernetesWorkflow var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace)); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; - }); + }, nameof(WaitUntilDeploymentOffline)); } private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment) @@ -896,10 +896,10 @@ namespace KubernetesWorkflow { var pods = FindPodsByLabel(deployment.PodLabel); return !pods.Any(); - }); + }, nameof(WaitUntilPodsForDeploymentAreOffline)); } - private void WaitUntil(Func predicate) + private void WaitUntil(Func predicate, string msg) { var sw = Stopwatch.Begin(log, true); try @@ -908,7 +908,7 @@ namespace KubernetesWorkflow } finally { - sw.End("", 1); + sw.End(msg, 1); } } From 015d8da21d8487e7eeabdff124458edcc919cb7c Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 09:17:25 +0200 Subject: [PATCH 07/22] Better logging for Time.WaitUntil. --- Framework/KubernetesWorkflow/K8sController.cs | 2 +- Framework/Utils/Time.cs | 13 ++++++++----- .../CodexContractsPlugin/CodexContractsStarter.cs | 8 ++++---- Tests/CodexTests/BasicTests/ContinuousSubstitute.cs | 2 +- Tests/CodexTests/BasicTests/NetworkIsolationTest.cs | 4 ++-- Tests/DistTestCore/Helpers/AssertHelpers.cs | 2 +- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index d81b87c4..71430a7e 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -904,7 +904,7 @@ namespace KubernetesWorkflow var sw = Stopwatch.Begin(log, true); try { - Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay()); + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg); } finally { diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index 82a836e6..e54ed184 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -57,24 +57,27 @@ return result; } - public static void WaitUntil(Func predicate) + public static void WaitUntil(Func predicate, string msg) { - WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg); } - public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay) + public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay, string msg) { var start = DateTime.UtcNow; + var tries = 1; var state = predicate(); while (!state) { - if (DateTime.UtcNow - start > timeout) + var duration = DateTime.UtcNow - start; + if (duration > timeout) { - throw new TimeoutException("Operation timed out."); + throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'"); } Sleep(retryDelay); state = predicate(); + tries++; } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 0137a173..d2b5c3f0 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -59,7 +59,7 @@ namespace CodexContractsPlugin var logHandler = new ContractsReadyLogHandler(tools.GetLog()); workflow.DownloadContainerLog(container, logHandler, 100); return logHandler.Found; - }); + }, nameof(DeployContract)); Log("Contracts deployed. Extracting addresses..."); var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container); @@ -71,7 +71,7 @@ namespace CodexContractsPlugin Log("Extract completed. Checking sync..."); - Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi)); + Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract)); Log("Synced. Codex SmartContracts deployed."); @@ -83,9 +83,9 @@ namespace CodexContractsPlugin tools.GetLog().Log(msg); } - private void WaitUntil(Func predicate) + private void WaitUntil(Func predicate, string msg) { - Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2)); + Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg); } private StartupConfig CreateStartupConfig(IGethNode gethNode) diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 2898be20..e135d2f4 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -154,7 +154,7 @@ namespace CodexTests.BasicTests var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value); return high > newBytes && newBytes > low; - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2)); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute)); FileUtils.TrackedFile? downloadedFile = null; LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId)); diff --git a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs index ccacdba4..26045504 100644 --- a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs @@ -19,7 +19,7 @@ namespace CodexTests.BasicTests { node = Ci.StartCodexNode(); - Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait)); } [Test] @@ -27,7 +27,7 @@ namespace CodexTests.BasicTests { var myNode = Ci.StartCodexNode(); - Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects)); try { diff --git a/Tests/DistTestCore/Helpers/AssertHelpers.cs b/Tests/DistTestCore/Helpers/AssertHelpers.cs index efd0749c..c0f995d3 100644 --- a/Tests/DistTestCore/Helpers/AssertHelpers.cs +++ b/Tests/DistTestCore/Helpers/AssertHelpers.cs @@ -14,7 +14,7 @@ namespace DistTestCore.Helpers Time.WaitUntil(() => { var c = constraint.Resolve(); return c.ApplyTo(actual()).IsSuccess; - }); + }, "RetryAssert: " + message); } catch (TimeoutException) { From 86074dab6a8ab72a7efde5054b9f84f3df222d4e Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 09:29:13 +0200 Subject: [PATCH 08/22] Bump k8s operation timeout for long timeset --- Framework/Core/TimeSet.cs | 10 +++++----- Tests/DistTestCore/Configuration.cs | 2 +- Tools/CodexNetDeployer/Deployer.cs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Framework/Core/TimeSet.cs b/Framework/Core/TimeSet.cs index 3acd8524..0b8ba94d 100644 --- a/Framework/Core/TimeSet.cs +++ b/Framework/Core/TimeSet.cs @@ -5,7 +5,7 @@ TimeSpan HttpCallTimeout(); int HttpMaxNumberOfRetries(); TimeSpan HttpCallRetryDelay(); - TimeSpan WaitForK8sServiceDelay(); + TimeSpan K8sOperationRetryDelay(); TimeSpan K8sOperationTimeout(); } @@ -26,7 +26,7 @@ return TimeSpan.FromSeconds(1); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { return TimeSpan.FromSeconds(10); } @@ -54,14 +54,14 @@ return TimeSpan.FromSeconds(2); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromSeconds(30); } public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(15); + return TimeSpan.FromHours(1); } } } diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index b1a94da5..39ae73ac 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -36,7 +36,7 @@ namespace DistTestCore var config = new KubernetesWorkflow.Configuration( kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay(), + retryDelay: timeSet.K8sOperationRetryDelay(), kubernetesNamespace: k8sNamespace ); diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 04171083..adea96d9 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -270,7 +270,7 @@ namespace CodexNetDeployer return TimeSpan.FromMinutes(10); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { return TimeSpan.FromSeconds(30); } From fb1090681635b32aae9d54114a00548c7254fa39 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 10:43:06 +0200 Subject: [PATCH 09/22] removes 20node value --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index b66345a3..364e582e 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -15,7 +15,7 @@ public class ScalabilityTests : CodexDistTest [Combinatorial] [UseLongTimeouts] public void ShouldMaintainFileInNetwork( - [Values(10, 20, 40, 80, 100)] int numberOfNodes, + [Values(10, 40, 80, 100)] int numberOfNodes, [Values(100, 1000, 5000, 10000)] int fileSizeInMb, [Values(true, false)] bool usePatchedImage ) From 3683044bf7dd87227923f9c76391b021c0d8ef66 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 11:17:59 +0200 Subject: [PATCH 10/22] Dont download container logs --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index 364e582e..4ce41bfb 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -14,6 +14,7 @@ public class ScalabilityTests : CodexDistTest [Test] [Combinatorial] [UseLongTimeouts] + [DontDownloadLogsOnFailure] public void ShouldMaintainFileInNetwork( [Values(10, 40, 80, 100)] int numberOfNodes, [Values(100, 1000, 5000, 10000)] int fileSizeInMb, From 23ebd4166b633eecb8614c4d06ee30c21ee7250c Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 15 Apr 2024 07:36:12 +0200 Subject: [PATCH 11/22] Disables downloading logs --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 2 +- Tests/DistTestCore/Configuration.cs | 3 +++ Tests/DistTestCore/DistTest.cs | 5 +++-- ...ogsOnFailureAttribute.cs => DontDownloadLogsAttribute.cs} | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) rename Tests/DistTestCore/{DontDownloadLogsOnFailureAttribute.cs => DontDownloadLogsAttribute.cs} (67%) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index 4ce41bfb..79fa1451 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -14,7 +14,7 @@ public class ScalabilityTests : CodexDistTest [Test] [Combinatorial] [UseLongTimeouts] - [DontDownloadLogsOnFailure] + [DontDownloadLogs] public void ShouldMaintainFileInNetwork( [Values(10, 40, 80, 100)] int numberOfNodes, [Values(100, 1000, 5000, 10000)] int fileSizeInMb, diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index 39ae73ac..3fa34b5e 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -24,6 +24,9 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; } + /// + /// Does not override [DontDownloadLogs] attribute. + /// public bool AlwaysDownloadContainerLogs { get; set; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace) diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 76f9d5bc..cfc31585 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -267,10 +267,11 @@ namespace DistTestCore private bool ShouldDownloadAllLogs(TestStatus testStatus) { + if (!IsDownloadingLogsEnabled()) return false; if (configuration.AlwaysDownloadContainerLogs) return true; if (testStatus == TestStatus.Failed) { - return IsDownloadingLogsEnabled(); + return true; } return false; @@ -289,7 +290,7 @@ namespace DistTestCore private bool IsDownloadingLogsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey); + return !testProperties.ContainsKey(DontDownloadLogsAttribute.DontDownloadKey); } } diff --git a/Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs b/Tests/DistTestCore/DontDownloadLogsAttribute.cs similarity index 67% rename from Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs rename to Tests/DistTestCore/DontDownloadLogsAttribute.cs index 800b35b8..13baab8f 100644 --- a/Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs +++ b/Tests/DistTestCore/DontDownloadLogsAttribute.cs @@ -3,11 +3,11 @@ namespace DistTestCore { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsOnFailureAttribute : PropertyAttribute + public class DontDownloadLogsAttribute : PropertyAttribute { public const string DontDownloadKey = "DontDownloadLogs"; - public DontDownloadLogsOnFailureAttribute() + public DontDownloadLogsAttribute() : base(DontDownloadKey) { } From 700fc0ea40297da1f49d369f646d3f1c31c806e3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 15 Apr 2024 07:57:13 +0200 Subject: [PATCH 12/22] Sets quota for codex nodes. Sets loglevel for bootstrap node. --- Framework/Utils/NumberSource.cs | 9 +++++++-- .../CodexTests/ScalabilityTests/ScalabilityTests.cs | 13 +++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Framework/Utils/NumberSource.cs b/Framework/Utils/NumberSource.cs index 2c7266fe..69d51899 100644 --- a/Framework/Utils/NumberSource.cs +++ b/Framework/Utils/NumberSource.cs @@ -2,6 +2,7 @@ { public class NumberSource { + private readonly object @lock = new object(); private int number; public NumberSource(int start) @@ -11,8 +12,12 @@ public int GetNextNumber() { - var n = number; - number++; + var n = -1; + lock (@lock) + { + n = number; + number++; + } return n; } } diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index 79fa1451..3fa7e213 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -23,9 +23,14 @@ public class ScalabilityTests : CodexDistTest { CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; - var bootstrap = AddCodex(); - var nodes = AddCodex(numberOfNodes - 1, - s => s.WithBootstrapNode(bootstrap).WithLogLevel(CodexLogLevel.Info)).ToList(); + var logLevel = CodexLogLevel.Info; + + var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); + var nodes = AddCodex(numberOfNodes - 1, s => s + .WithBootstrapNode(bootstrap) + .WithLogLevel(logLevel) + .WithStorageQuota((fileSizeInMb + 50).MB()) + ).ToList(); var uploader = nodes.PickOneRandom(); var downloader = nodes.PickOneRandom(); @@ -43,4 +48,4 @@ public class ScalabilityTests : CodexDistTest downloadedFile!.AssertIsEqual(testFile); } -} \ No newline at end of file +} From 570b174a00020585c295b68efd8e6c88ddb51f2f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 15 Apr 2024 08:12:57 +0200 Subject: [PATCH 13/22] Adds everyone-test-a-file test --- .../ScalabilityTests/ScalabilityTests.cs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index 3fa7e213..0940e34a 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -1,5 +1,6 @@ using CodexPlugin; using DistTestCore; +using FileUtils; using NUnit.Framework; using Utils; @@ -11,6 +12,10 @@ public class ScalabilityTests : CodexDistTest private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests"; private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests"; + /// + /// We upload a file to node A, then download it with B. + /// Then we stop node A, and download again with node C. + /// [Test] [Combinatorial] [UseLongTimeouts] @@ -48,4 +53,76 @@ public class ScalabilityTests : CodexDistTest downloadedFile!.AssertIsEqual(testFile); } + + /// + /// We upload a file to each node, to put a more wide-spread load on the network. + /// Then we run the same test as ShouldMaintainFileInNetwork. + /// + [Test] + [Combinatorial] + [UseLongTimeouts] + [DontDownloadLogs] + public void EveryoneGetsAFile( + [Values(10, 40, 80, 100)] int numberOfNodes, + [Values(100, 1000)] int fileSizeInMb, + [Values(true, false)] bool usePatchedImage + ) + { + CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; + + var logLevel = CodexLogLevel.Info; + + var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); + var nodes = AddCodex(numberOfNodes - 1, s => s + .WithBootstrapNode(bootstrap) + .WithLogLevel(logLevel) + .WithStorageQuota((fileSizeInMb + 50).MB()) + ).ToList(); + + var pairTasks = nodes.Select(n => + { + return Task.Run(() => + { + var file = GenerateTestFile(fileSizeInMb.MB()); + var cid = n.UploadFile(file); + return new NodeFilePair(n, file, cid); + }); + }); + + var pairs = pairTasks.Select(t => Time.Wait(t)).ToList(); + + RunDoubleDownloadTest( + pairs.PickOneRandom(), + pairs.PickOneRandom(), + pairs.PickOneRandom() + ); + } + + private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2) + { + var expectedFile = source.File; + var cid = source.Cid; + + var file1 = dl1.Node.DownloadContent(cid); + file1!.AssertIsEqual(expectedFile); + + source.Node.Stop(true); + + var file2 = dl2.Node.DownloadContent(cid); + file2!.AssertIsEqual(expectedFile); + } + + public class NodeFilePair + { + public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid) + { + Node = node; + File = file; + Cid = cid; + } + + public ICodexNode Node { get; } + public TrackedFile File { get; } + public ContentId Cid { get; } + } } From 630dc2814ae0e2ce1d121b17fbd2aaa8fa65297d Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 15 Apr 2024 08:13:43 +0200 Subject: [PATCH 14/22] disable new test --- Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs index 0940e34a..598f0bf2 100644 --- a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -58,6 +58,7 @@ public class ScalabilityTests : CodexDistTest /// We upload a file to each node, to put a more wide-spread load on the network. /// Then we run the same test as ShouldMaintainFileInNetwork. /// + [Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")] [Test] [Combinatorial] [UseLongTimeouts] From eed989cbf52398a0c3de1b0f6a5f5f7a1ae740ea Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 15 Apr 2024 11:37:14 +0200 Subject: [PATCH 15/22] Handle errors during log download --- Tests/DistTestCore/DistTest.cs | 2 +- Tests/DistTestCore/TestLifecycle.cs | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index cfc31585..b6c0ba96 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -98,7 +98,7 @@ namespace DistTestCore } catch (Exception ex) { - fixtureLog.Error("Cleanup failed: " + ex.Message); + fixtureLog.Error("Cleanup failed: " + ex); GlobalTestFailure.HasFailed = true; } } diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 1421fe59..542ca27c 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -93,13 +93,20 @@ namespace DistTestCore public void DownloadAllLogs() { - foreach (var rc in runningContainers) + try { - foreach (var c in rc.Containers) + foreach (var rc in runningContainers) { - CoreInterface.DownloadLog(c); + foreach (var c in rc.Containers) + { + CoreInterface.DownloadLog(c); + } } } + catch (Exception ex) + { + Log.Error("Exception during log download: " + ex); + } } } } From c856f404e3fe8753e70c94a965c2419b4d523f96 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 19 Apr 2024 11:40:32 +0200 Subject: [PATCH 16/22] Adds test to show from which hosts blocks are downloaded. --- Framework/Core/DownloadedLog.cs | 14 ++++ .../MultiPeerDownloadTests.cs | 72 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs diff --git a/Framework/Core/DownloadedLog.cs b/Framework/Core/DownloadedLog.cs index 923bcbc8..fa4d5597 100644 --- a/Framework/Core/DownloadedLog.cs +++ b/Framework/Core/DownloadedLog.cs @@ -4,6 +4,7 @@ namespace Core { public interface IDownloadedLog { + void IterateLines(Action action); string[] GetLinesContaining(string expectedString); string[] FindLinesThatContain(params string[] tags); void DeleteFile(); @@ -18,6 +19,19 @@ namespace Core this.logFile = logFile; } + public void IterateLines(Action action) + { + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + action(line); + line = streamReader.ReadLine(); + } + } + public string[] GetLinesContaining(string expectedString) { using var file = File.OpenRead(logFile.FullFilename); diff --git a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs new file mode 100644 index 00000000..2ff5103d --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -0,0 +1,72 @@ +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class MultiPeerDownloadTests : AutoBootstrapDistTest + { + [Test] + public void MultiPeerDownload() + { + var hosts = AddCodex(5, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var file = GenerateTestFile(100.MB()); + var cid = hosts[0].UploadFile(file); + + var uploadLog = Ci.DownloadLog(hosts[0]); + var blockCids = uploadLog + .FindLinesThatContain("Putting block into network store") + .Select(s => + { + var start = s.IndexOf("cid=") + 4; + var end = s.IndexOf(" count="); + var len = end - start; + return s.Substring(start, len); + }) + .ToArray(); + + // Each host has the file. + foreach (var h in hosts) h.DownloadContent(cid); + + var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var resultFile = client.DownloadContent(cid); + resultFile!.AssertIsEqual(file); + + var downloadLog = Ci.DownloadLog(client); + var blocksPerHost = new Dictionary(); + var seenBlocks = new List(); + var host = string.Empty; + downloadLog.IterateLines(line => + { + if (line.Contains("peer=") && line.Contains(" len=")) + { + var start = line.IndexOf("peer=") + 5; + var end = line.IndexOf(" len="); + var len = end - start; + host = line.Substring(start, len); + } + else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key")) + { + var start = line.IndexOf("cid=") + 4; + var end = line.IndexOf(" count="); + var len = end - start; + var blockCid = line.Substring(start, len); + + if (!seenBlocks.Contains(blockCid)) + { + seenBlocks.Add(blockCid); + if (!blocksPerHost.ContainsKey(host)) blocksPerHost.Add(host, 1); + else blocksPerHost[host]++; + } + } + }); + + Log("Total number of blocks in dataset: " + blockCids.Length); + Log("Blocks fetched per host:"); + foreach (var pair in blocksPerHost) + { + Log($"Host: {pair.Key} = {pair.Value}"); + } + } + } +} From c4c3f61a23488b9773b697c14ffb5b92048f80fa Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 19 Apr 2024 11:52:39 +0200 Subject: [PATCH 17/22] parameterizes tests --- ProjectPlugins/CodexPlugin/CodexNode.cs | 2 +- .../ScalabilityTests/MultiPeerDownloadTests.cs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 38b8d418..36fb1dc0 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -73,7 +73,7 @@ namespace CodexPlugin public string GetName() { - return CodexAccess.Container.Name; + return Container.Name; } public DebugInfo GetDebugInfo() diff --git a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs index 2ff5103d..3c4e4822 100644 --- a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs +++ b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -7,10 +7,14 @@ namespace CodexTests.ScalabilityTests public class MultiPeerDownloadTests : AutoBootstrapDistTest { [Test] - public void MultiPeerDownload() + [Combinatorial] + public void MultiPeerDownload( + [Values(5, 10, 20)] int numberOfHosts, + [Values(100, 1000)] int fileSize + ) { - var hosts = AddCodex(5, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); - var file = GenerateTestFile(100.MB()); + var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var file = GenerateTestFile(fileSize.MB()); var cid = hosts[0].UploadFile(file); var uploadLog = Ci.DownloadLog(hosts[0]); From 7ec99347515d532034efacdf58d2381fda5d16ff Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 22 Apr 2024 11:17:47 +0200 Subject: [PATCH 18/22] Update to multipeer download test --- .../MultiPeerDownloadTests.cs | 68 +++++++++++++++---- Tests/DistTestCore/DistTest.cs | 19 ++++-- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs index 3c4e4822..f4c0a5f0 100644 --- a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs +++ b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using DistTestCore; +using NUnit.Framework; using Utils; namespace CodexTests.ScalabilityTests @@ -7,6 +8,8 @@ namespace CodexTests.ScalabilityTests public class MultiPeerDownloadTests : AutoBootstrapDistTest { [Test] + [DontDownloadLogs] + [UseLongTimeouts] [Combinatorial] public void MultiPeerDownload( [Values(5, 10, 20)] int numberOfHosts, @@ -16,8 +19,10 @@ namespace CodexTests.ScalabilityTests var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); var file = GenerateTestFile(fileSize.MB()); var cid = hosts[0].UploadFile(file); + var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6); var uploadLog = Ci.DownloadLog(hosts[0]); + var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block. var blockCids = uploadLog .FindLinesThatContain("Putting block into network store") .Select(s => @@ -29,7 +34,8 @@ namespace CodexTests.ScalabilityTests }) .ToArray(); - // Each host has the file. + Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks)); + foreach (var h in hosts) h.DownloadContent(cid); var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); @@ -37,9 +43,8 @@ namespace CodexTests.ScalabilityTests resultFile!.AssertIsEqual(file); var downloadLog = Ci.DownloadLog(client); - var blocksPerHost = new Dictionary(); - var seenBlocks = new List(); var host = string.Empty; + var blockCidHostMap = new Dictionary(); downloadLog.IterateLines(line => { if (line.Contains("peer=") && line.Contains(" len=")) @@ -56,21 +61,60 @@ namespace CodexTests.ScalabilityTests var len = end - start; var blockCid = line.Substring(start, len); - if (!seenBlocks.Contains(blockCid)) - { - seenBlocks.Add(blockCid); - if (!blocksPerHost.ContainsKey(host)) blocksPerHost.Add(host, 1); - else blocksPerHost[host]++; - } + blockCidHostMap.Add(blockCid, host); + host = string.Empty; } }); - Log("Total number of blocks in dataset: " + blockCids.Length); + var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value)); + //PrintFullMap(blockCidHostMap); + PrintOverview(blockCidHostMap); + + Log("Expected number of blocks: " + expectedNumberOfBlocks); + Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length); + Log("Total blocks fetched by hosts: " + totalFetched); + Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks)); + } + + private void PrintOverview(Dictionary blockCidHostMap) + { + var overview = new Dictionary(); + foreach (var pair in blockCidHostMap) + { + if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1); + else overview[pair.Value]++; + } + Log("Blocks fetched per host:"); - foreach (var pair in blocksPerHost) + foreach (var pair in overview) { Log($"Host: {pair.Key} = {pair.Value}"); } } + + private void PrintFullMap(Dictionary blockCidHostMap) + { + Log("Per block, host it was fetched from:"); + foreach (var pair in blockCidHostMap) + { + if (string.IsNullOrEmpty(pair.Value)) + { + Log($"block: {pair.Key} = Not seen"); + } + else + { + Log($"block: {pair.Key} = '{pair.Value}'"); + } + } + } + + private long RoundUp(long filesize, long blockSize) + { + double f = filesize; + double b = blockSize; + + var result = Math.Ceiling(f / b); + return Convert.ToInt64(result); + } } } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index b6c0ba96..ed99fe94 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -236,9 +236,19 @@ namespace DistTestCore } private bool ShouldUseLongTimeouts() + { + return CurrentTestMethodHasAttribute(); + } + + private bool HasDontDownloadAttribute() + { + return CurrentTestMethodHasAttribute(); + } + + private bool CurrentTestMethodHasAttribute() where T : PropertyAttribute { // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test. - // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out + // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out // if the attribute is present. var currentTest = TestContext.CurrentContext.Test; var className = currentTest.ClassName; @@ -247,7 +257,7 @@ namespace DistTestCore var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); - return testMethods.Any(m => m.GetCustomAttribute() != null); + return testMethods.Any(m => m.GetCustomAttribute() != null); } private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) @@ -267,8 +277,8 @@ namespace DistTestCore private bool ShouldDownloadAllLogs(TestStatus testStatus) { - if (!IsDownloadingLogsEnabled()) return false; if (configuration.AlwaysDownloadContainerLogs) return true; + if (!IsDownloadingLogsEnabled()) return false; if (testStatus == TestStatus.Failed) { return true; @@ -289,8 +299,7 @@ namespace DistTestCore private bool IsDownloadingLogsEnabled() { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(DontDownloadLogsAttribute.DontDownloadKey); + return !HasDontDownloadAttribute(); } } From 58f7f9384af4ef757c93276be941da354cc821d3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 23 Apr 2024 09:10:10 +0200 Subject: [PATCH 19/22] Test for disc speeds --- Framework/Utils/Formatter.cs | 6 +- .../ScalabilityTests/ClusterSpeedTests.cs | 83 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs diff --git a/Framework/Utils/Formatter.cs b/Framework/Utils/Formatter.cs index 1ea15503..7422c84e 100644 --- a/Framework/Utils/Formatter.cs +++ b/Framework/Utils/Formatter.cs @@ -1,4 +1,6 @@ -namespace Utils +using System.Globalization; + +namespace Utils { public static class Formatter { @@ -10,7 +12,7 @@ var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1); - return digit.ToString() + sizeSuffixes[sizeOrder]; + return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder]; } } } diff --git a/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs new file mode 100644 index 00000000..d8b27b10 --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs @@ -0,0 +1,83 @@ +using DistTestCore; +using Logging; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class ClusterDiscSpeedTests : DistTest + { + private readonly Random random = new Random(); + + [Test] + [Combinatorial] + public void DiscSpeedTest( + [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb + ) + { + long targetSize = (long)(1024 * 1024 * 1024) * 2; + long bufferSizeBytes = ((long)bufferSizeKb) * 1024; + + var filename = nameof(DiscSpeedTest); + + Thread.Sleep(1000); + if (File.Exists(filename)) File.Delete(filename); + Thread.Sleep(1000); + var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename); + Thread.Sleep(1000); + var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename); + + Log($"Write speed: {writeSpeed} per second."); + Log($"Read speed: {writeSpeed} per second."); + } + + private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename) + { + long bytesWritten = 0; + var buffer = new byte[bufferSizeBytes]; + random.NextBytes(buffer); + + var sw = Stopwatch.Begin(GetTestLog()); + using (var stream = File.OpenWrite(filename)) + { + while (bytesWritten < targetSize) + { + long remaining = targetSize - bytesWritten; + long toWrite = Math.Min(bufferSizeBytes, remaining); + + stream.Write(buffer, 0, Convert.ToInt32(toWrite)); + bytesWritten += toWrite; + } + } + var duration = sw.End("WriteTime"); + double totalSeconds = duration.TotalSeconds; + double totalBytes = bytesWritten; + double bytesPerSecond = totalBytes / totalSeconds; + return new ByteSize(Convert.ToInt64(bytesPerSecond)); + } + + private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename) + { + long bytesRead = 0; + var buffer = new byte[bufferSizeBytes]; + var sw = Stopwatch.Begin(GetTestLog()); + using (var stream = File.OpenRead(filename)) + { + while (bytesRead < targetSize) + { + long remaining = targetSize - bytesRead; + long toRead = Math.Min(bufferSizeBytes, remaining); + + var r = stream.Read(buffer, 0, Convert.ToInt32(toRead)); + bytesRead += r; + } + } + var duration = sw.End("ReadTime"); + double totalSeconds = duration.TotalSeconds; + double totalBytes = bytesRead; + double bytesPerSecond = totalBytes / totalSeconds; + return new ByteSize(Convert.ToInt64(bytesPerSecond)); + } + } +} From 6545f3469f54b72d05aeb3f1a8aad3ef23355270 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 23 Apr 2024 13:16:11 +0200 Subject: [PATCH 20/22] dumb mistake by me --- Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs index d8b27b10..0e3fdafd 100644 --- a/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs +++ b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs @@ -21,15 +21,15 @@ namespace CodexTests.ScalabilityTests var filename = nameof(DiscSpeedTest); - Thread.Sleep(1000); + Thread.Sleep(2000); if (File.Exists(filename)) File.Delete(filename); - Thread.Sleep(1000); + Thread.Sleep(2000); var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename); - Thread.Sleep(1000); + Thread.Sleep(2000); var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename); Log($"Write speed: {writeSpeed} per second."); - Log($"Read speed: {writeSpeed} per second."); + Log($"Read speed: {readSpeed} per second."); } private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename) From a6379d02f1f124ccff2a99787123a45d3f87196d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 24 Apr 2024 09:41:05 +0200 Subject: [PATCH 21/22] Adds test for upload/download large file with single node --- .../OneClientLargeFileTests.cs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs diff --git a/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs new file mode 100644 index 00000000..76c8fdff --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs @@ -0,0 +1,36 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class OneClientLargeFileTests : CodexDistTest + { + [Test] + [Combinatorial] + [UseLongTimeouts] + public void OneClientLargeFile([Values( + 256, + 512, + 1024, // GB + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072 + )] int sizeMb) + { + var testFile = GenerateTestFile(sizeMb.MB()); + + var node = AddCodex(s => s.WithLogLevel(CodexLogLevel.Warn)); + var contentId = node.UploadFile(testFile); + var downloadedFile = node.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} From 50b7e2300db3180ac0307d5d67a3043a815ed4bd Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 25 Apr 2024 09:25:00 +0200 Subject: [PATCH 22/22] sets quota --- Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs index 76c8fdff..75dccbe3 100644 --- a/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs +++ b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs @@ -26,7 +26,10 @@ namespace CodexTests.ScalabilityTests { var testFile = GenerateTestFile(sizeMb.MB()); - var node = AddCodex(s => s.WithLogLevel(CodexLogLevel.Warn)); + var node = AddCodex(s => s + .WithLogLevel(CodexLogLevel.Warn) + .WithStorageQuota((sizeMb + 10).MB()) + ); var contentId = node.UploadFile(testFile); var downloadedFile = node.DownloadContent(contentId);