diff --git a/Framework/Core/DownloadedLog.cs b/Framework/Core/DownloadedLog.cs index 923bcbc..fa4d559 100644 --- a/Framework/Core/DownloadedLog.cs +++ b/Framework/Core/DownloadedLog.cs @@ -4,6 +4,7 @@ namespace Core { public interface IDownloadedLog { + void IterateLines(Action action); string[] GetLinesContaining(string expectedString); string[] FindLinesThatContain(params string[] tags); void DeleteFile(); @@ -18,6 +19,19 @@ namespace Core this.logFile = logFile; } + public void IterateLines(Action action) + { + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + action(line); + line = streamReader.ReadLine(); + } + } + public string[] GetLinesContaining(string expectedString) { using var file = File.OpenRead(logFile.FullFilename); diff --git a/Framework/Core/TimeSet.cs b/Framework/Core/TimeSet.cs index 3acd852..0b8ba94 100644 --- a/Framework/Core/TimeSet.cs +++ b/Framework/Core/TimeSet.cs @@ -5,7 +5,7 @@ TimeSpan HttpCallTimeout(); int HttpMaxNumberOfRetries(); TimeSpan HttpCallRetryDelay(); - TimeSpan WaitForK8sServiceDelay(); + TimeSpan K8sOperationRetryDelay(); TimeSpan K8sOperationTimeout(); } @@ -26,7 +26,7 @@ return TimeSpan.FromSeconds(1); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { return TimeSpan.FromSeconds(10); } @@ -54,14 +54,14 @@ return TimeSpan.FromSeconds(2); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromSeconds(30); } public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(15); + return TimeSpan.FromHours(1); } } } diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 80b15d1..71430a7 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -705,7 +705,7 @@ namespace KubernetesWorkflow private string GetPodName(RunningContainer container) { - return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name; + return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name; } private V1Pod GetPodForDeployment(RunningDeployment deployment) @@ -868,7 +868,7 @@ namespace KubernetesWorkflow private void WaitUntilNamespaceCreated() { - WaitUntil(() => IsNamespaceOnline(K8sNamespace)); + WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated)); } private void WaitUntilDeploymentOnline(string deploymentName) @@ -877,7 +877,7 @@ namespace KubernetesWorkflow { var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace)); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; - }); + }, nameof(WaitUntilDeploymentOnline)); } private void WaitUntilDeploymentOffline(string deploymentName) @@ -887,7 +887,7 @@ namespace KubernetesWorkflow var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace)); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; - }); + }, nameof(WaitUntilDeploymentOffline)); } private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment) @@ -896,19 +896,19 @@ namespace KubernetesWorkflow { var pods = FindPodsByLabel(deployment.PodLabel); return !pods.Any(); - }); + }, nameof(WaitUntilPodsForDeploymentAreOffline)); } - private void WaitUntil(Func predicate) + private void WaitUntil(Func predicate, string msg) { var sw = Stopwatch.Begin(log, true); try { - Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay()); + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg); } finally { - sw.End("", 1); + sw.End(msg, 1); } } diff --git a/Framework/KubernetesWorkflow/K8sHooks.cs b/Framework/KubernetesWorkflow/K8sHooks.cs index 74bb93b..4dad740 100644 --- a/Framework/KubernetesWorkflow/K8sHooks.cs +++ b/Framework/KubernetesWorkflow/K8sHooks.cs @@ -5,18 +5,18 @@ namespace KubernetesWorkflow { public interface IK8sHooks { - void OnContainersStarted(RunningContainers runningContainers); - void OnContainersStopped(RunningContainers runningContainers); + void OnContainersStarted(RunningPod runningPod); + void OnContainersStopped(RunningPod runningPod); void OnContainerRecipeCreated(ContainerRecipe recipe); } public class DoNothingK8sHooks : IK8sHooks { - public void OnContainersStarted(RunningContainers runningContainers) + public void OnContainersStarted(RunningPod runningPod) { } - public void OnContainersStopped(RunningContainers runningContainers) + public void OnContainersStopped(RunningPod runningPod) { } diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index b8f7d77..cda6148 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -12,9 +12,9 @@ namespace KubernetesWorkflow FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); PodInfo GetPodInfo(RunningContainer container); - PodInfo GetPodInfo(RunningContainers containers); + PodInfo GetPodInfo(RunningPod pod); CrashWatcher CreateCrashWatcher(RunningContainer container); - void Stop(RunningContainers containers, bool waitTillStopped); + void Stop(RunningPod pod, bool waitTillStopped); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); void DeleteNamespace(); @@ -60,7 +60,7 @@ namespace KubernetesWorkflow var startResult = controller.BringOnline(recipes, location); var containers = CreateContainers(startResult, recipes, startupConfig); - var rc = new RunningContainers(startupConfig, startResult, containers); + var rc = new RunningPod(startupConfig, startResult, containers); cluster.Configuration.Hooks.OnContainersStarted(rc); if (startResult.ExternalService != null) @@ -71,7 +71,7 @@ namespace KubernetesWorkflow }); } - public void WaitUntilOnline(RunningContainers rc) + public void WaitUntilOnline(RunningPod rc) { K8s(controller => { @@ -84,12 +84,12 @@ namespace KubernetesWorkflow public PodInfo GetPodInfo(RunningContainer container) { - return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment)); + return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment)); } - public PodInfo GetPodInfo(RunningContainers containers) + public PodInfo GetPodInfo(RunningPod pod) { - return K8s(c => c.GetPodInfo(containers.StartResult.Deployment)); + return K8s(c => c.GetPodInfo(pod.StartResult.Deployment)); } public CrashWatcher CreateCrashWatcher(RunningContainer container) @@ -97,12 +97,12 @@ namespace KubernetesWorkflow return K8s(c => c.CreateCrashWatcher(container)); } - public void Stop(RunningContainers runningContainers, bool waitTillStopped) + public void Stop(RunningPod runningPod, bool waitTillStopped) { K8s(controller => { - controller.Stop(runningContainers.StartResult, waitTillStopped); - cluster.Configuration.Hooks.OnContainersStopped(runningContainers); + controller.Stop(runningPod.StartResult, waitTillStopped); + cluster.Configuration.Hooks.OnContainersStopped(runningPod); }); } diff --git a/Framework/KubernetesWorkflow/Types/FutureContainers.cs b/Framework/KubernetesWorkflow/Types/FutureContainers.cs index 262eac4..296be53 100644 --- a/Framework/KubernetesWorkflow/Types/FutureContainers.cs +++ b/Framework/KubernetesWorkflow/Types/FutureContainers.cs @@ -2,19 +2,19 @@ { public class FutureContainers { - private readonly RunningContainers runningContainers; + private readonly RunningPod runningPod; private readonly StartupWorkflow workflow; - public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow) + public FutureContainers(RunningPod runningPod, StartupWorkflow workflow) { - this.runningContainers = runningContainers; + this.runningPod = runningPod; this.workflow = workflow; } - public RunningContainers WaitForOnline() + public RunningPod WaitForOnline() { - workflow.WaitUntilOnline(runningContainers); - return runningContainers; + workflow.WaitUntilOnline(runningPod); + return runningPod; } } } diff --git a/Framework/KubernetesWorkflow/Types/RunningContainer.cs b/Framework/KubernetesWorkflow/Types/RunningContainer.cs index f391e45..b0fbc02 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainer.cs +++ b/Framework/KubernetesWorkflow/Types/RunningContainer.cs @@ -19,7 +19,7 @@ namespace KubernetesWorkflow.Types public ContainerAddress[] Addresses { get; } [JsonIgnore] - public RunningContainers RunningContainers { get; internal set; } = null!; + public RunningPod RunningPod { get; internal set; } = null!; public Address GetAddress(ILog log, string portTag) { diff --git a/Framework/KubernetesWorkflow/Types/RunningContainers.cs b/Framework/KubernetesWorkflow/Types/RunningPod.cs similarity index 61% rename from Framework/KubernetesWorkflow/Types/RunningContainers.cs rename to Framework/KubernetesWorkflow/Types/RunningPod.cs index 9a6e5f3..7f1a24d 100644 --- a/Framework/KubernetesWorkflow/Types/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/Types/RunningPod.cs @@ -2,15 +2,15 @@ namespace KubernetesWorkflow.Types { - public class RunningContainers + public class RunningPod { - public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) + public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers) { StartupConfig = startupConfig; StartResult = startResult; Containers = containers; - foreach (var c in containers) c.RunningContainers = this; + foreach (var c in containers) c.RunningPod = this; } public StartupConfig StartupConfig { get; } @@ -31,12 +31,7 @@ namespace KubernetesWorkflow.Types public static class RunningContainersExtensions { - public static RunningContainer[] Containers(this RunningContainers[] runningContainers) - { - return runningContainers.SelectMany(c => c.Containers).ToArray(); - } - - public static string Describe(this RunningContainers[] runningContainers) + public static string Describe(this RunningPod[] runningContainers) { return string.Join(",", runningContainers.Select(c => c.Describe())); } diff --git a/Framework/Utils/Formatter.cs b/Framework/Utils/Formatter.cs index 1ea1550..7422c84 100644 --- a/Framework/Utils/Formatter.cs +++ b/Framework/Utils/Formatter.cs @@ -1,4 +1,6 @@ -namespace Utils +using System.Globalization; + +namespace Utils { public static class Formatter { @@ -10,7 +12,7 @@ var sizeOrder = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); var digit = Math.Round(bytes / Math.Pow(1024, sizeOrder), 1); - return digit.ToString() + sizeSuffixes[sizeOrder]; + return digit.ToString(CultureInfo.InvariantCulture) + sizeSuffixes[sizeOrder]; } } } diff --git a/Framework/Utils/NumberSource.cs b/Framework/Utils/NumberSource.cs index 2c7266f..69d5189 100644 --- a/Framework/Utils/NumberSource.cs +++ b/Framework/Utils/NumberSource.cs @@ -2,6 +2,7 @@ { public class NumberSource { + private readonly object @lock = new object(); private int number; public NumberSource(int start) @@ -11,8 +12,12 @@ public int GetNextNumber() { - var n = number; - number++; + var n = -1; + lock (@lock) + { + n = number; + number++; + } return n; } } diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index 82a836e..e54ed18 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -57,24 +57,27 @@ return result; } - public static void WaitUntil(Func predicate) + public static void WaitUntil(Func predicate, string msg) { - WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg); } - public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay) + public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay, string msg) { var start = DateTime.UtcNow; + var tries = 1; var state = predicate(); while (!state) { - if (DateTime.UtcNow - start > timeout) + var duration = DateTime.UtcNow - start; + if (duration > timeout) { - throw new TimeoutException("Operation timed out."); + throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'"); } Sleep(retryDelay); state = predicate(); + tries++; } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 0137a17..d2b5c3f 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -59,7 +59,7 @@ namespace CodexContractsPlugin var logHandler = new ContractsReadyLogHandler(tools.GetLog()); workflow.DownloadContainerLog(container, logHandler, 100); return logHandler.Found; - }); + }, nameof(DeployContract)); Log("Contracts deployed. Extracting addresses..."); var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container); @@ -71,7 +71,7 @@ namespace CodexContractsPlugin Log("Extract completed. Checking sync..."); - Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi)); + Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract)); Log("Synced. Codex SmartContracts deployed."); @@ -83,9 +83,9 @@ namespace CodexContractsPlugin tools.GetLog().Log(msg); } - private void WaitUntil(Func predicate) + private void WaitUntil(Func predicate, string msg) { - Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2)); + Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg); } private StartupConfig CreateStartupConfig(IGethNode gethNode) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index fd082cf..f347805 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -29,19 +29,19 @@ namespace CodexDiscordBotPlugin { } - public RunningContainers Deploy(DiscordBotStartupConfig config) + public RunningPod Deploy(DiscordBotStartupConfig config) { var workflow = tools.CreateWorkflow(); return StartContainer(workflow, config); } - public RunningContainers DeployRewarder(RewarderBotStartupConfig config) + public RunningPod DeployRewarder(RewarderBotStartupConfig config) { var workflow = tools.CreateWorkflow(); return StartRewarderContainer(workflow, config); } - private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config) + private RunningPod StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; @@ -49,7 +49,7 @@ namespace CodexDiscordBotPlugin return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); } - private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) + private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.Add(config); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs index c17cec1..c91d711 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs @@ -5,12 +5,12 @@ namespace CodexDiscordBotPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) + public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) { return Plugin(ci).Deploy(config); } - public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config) + public static RunningPod DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config) { return Plugin(ci).DeployRewarder(config); } diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 212fa19..5cf3dc9 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -38,7 +38,7 @@ namespace CodexPlugin if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger."); } - public void CheckCompatibility(RunningContainers[] containers) + public void CheckCompatibility(RunningPod[] containers) { if (checkPassed) return; diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 697ff2c..e35a9f9 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -13,7 +13,7 @@ namespace CodexPlugin private readonly Mapper mapper = new Mapper(); private bool hasContainerCrashed; - public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher) + public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher) { this.tools = tools; Container = container; @@ -23,7 +23,7 @@ namespace CodexPlugin CrashWatcher.Start(this); } - public RunningContainer Container { get; } + public RunningPod Container { get; } public CrashWatcher CrashWatcher { get; } public DebugInfo GetDebugInfo() @@ -136,7 +136,7 @@ namespace CodexPlugin private Address GetAddress() { - return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); + return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); } private void CheckContainerCrashed(HttpClient client) diff --git a/ProjectPlugins/CodexPlugin/CodexDeployment.cs b/ProjectPlugins/CodexPlugin/CodexDeployment.cs index 9f74dd3..7cbf450 100644 --- a/ProjectPlugins/CodexPlugin/CodexDeployment.cs +++ b/ProjectPlugins/CodexPlugin/CodexDeployment.cs @@ -7,8 +7,8 @@ namespace CodexPlugin public class CodexDeployment { public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, - CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer, - RunningContainers? discordBotContainer, DeploymentMetadata metadata, + CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer, + RunningPod? discordBotContainer, DeploymentMetadata metadata, String id) { Id = id; @@ -24,20 +24,20 @@ namespace CodexPlugin public CodexInstance[] CodexInstances { get; } public GethDeployment GethDeployment { get; } public CodexContractsDeployment CodexContractsDeployment { get; } - public RunningContainers? PrometheusContainer { get; } - public RunningContainers? DiscordBotContainer { get; } + public RunningPod? PrometheusContainer { get; } + public RunningPod? DiscordBotContainer { get; } public DeploymentMetadata Metadata { get; } } public class CodexInstance { - public CodexInstance(RunningContainers containers, DebugInfo info) + public CodexInstance(RunningPod pod, DebugInfo info) { - Containers = containers; + Pod = pod; Info = info; } - public RunningContainers Containers { get; } + public RunningPod Pod { get; } public DebugInfo Info { get; } } diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index e1e475c..36fb1dc 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -44,7 +44,9 @@ namespace CodexPlugin transferSpeeds = new TransferSpeeds(); } - public RunningContainer Container { get { return CodexAccess.Container; } } + public RunningPod Pod { get { return CodexAccess.Container; } } + + public RunningContainer Container { get { return Pod.Containers.Single(); } } public CodexAccess CodexAccess { get; } public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; } public CodexNodeGroup Group { get; } @@ -56,7 +58,7 @@ namespace CodexPlugin { get { - return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag); + return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag); } } @@ -71,7 +73,7 @@ namespace CodexPlugin public string GetName() { - return CodexAccess.Container.Name; + return Container.Name; } public DebugInfo GetDebugInfo() @@ -142,11 +144,13 @@ namespace CodexPlugin public void Stop(bool waitTillStopped) { - if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + - "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + - "available for codex-nodes in groups of 1."); - - Group.BringOffline(waitTillStopped); + CrashWatcher.Stop(); + Group.Stop(this, waitTillStopped); + // if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " + + // "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " + + // "available for codex-nodes in groups of 1."); + // + // Group.BringOffline(waitTillStopped); } public void EnsureOnlineGetVersionResponse() @@ -171,7 +175,7 @@ namespace CodexPlugin // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. var workflow = tools.CreateWorkflow(); - var podInfo = workflow.GetPodInfo(peer.Container); + var podInfo = workflow.GetPodInfo(peer.Pod); return peerInfo.Addrs.Select(a => a .Replace("0.0.0.0", podInfo.Ip)) diff --git a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs index 05b80b2..18483de 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs @@ -35,7 +35,7 @@ namespace CodexPlugin private EthAddress? GetEthAddress(CodexAccess access) { - var ethAccount = access.Container.Recipe.Additionals.Get(); + var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get(); if (ethAccount == null) return null; return ethAccount.EthAddress; } diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index 1f7ed4e..47bf35d 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -15,11 +15,11 @@ namespace CodexPlugin { private readonly CodexStarter starter; - public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory) + public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory) { this.starter = starter; Containers = containers; - Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); + Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); Version = new DebugInfoVersion(); } @@ -39,7 +39,14 @@ namespace CodexPlugin Containers = null!; } - public RunningContainers[] Containers { get; private set; } + public void Stop(CodexNode node, bool waitTillStopped) + { + starter.Stop(node.Pod, waitTillStopped); + Nodes = Nodes.Where(n => n != node).ToArray(); + Containers = Containers.Where(c => c != node.Pod).ToArray(); + } + + public RunningPod[] Containers { get; private set; } public CodexNode[] Nodes { get; private set; } public DebugInfoVersion Version { get; private set; } public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray(); @@ -74,9 +81,9 @@ namespace CodexPlugin Version = first; } - private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory) + private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory) { - var watcher = factory.CreateCrashWatcher(c); + var watcher = factory.CreateCrashWatcher(c.Containers.Single()); var access = new CodexAccess(tools, c, watcher); return factory.CreateOnlineCodexNode(access, this); } diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 9b8586e..7b722ed 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -32,13 +32,13 @@ namespace CodexPlugin { } - public RunningContainers[] DeployCodexNodes(int numberOfNodes, Action setup) + public RunningPod[] DeployCodexNodes(int numberOfNodes, Action setup) { var codexSetup = GetSetup(numberOfNodes, setup); return codexStarter.BringOnline(codexSetup); } - public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers) + public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers) { containers = containers.Select(c => SerializeGate.Gate(c)).ToArray(); return codexStarter.WrapCodexContainers(coreInterface, containers); diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 6fcc4bd..dec13d3 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -19,7 +19,7 @@ namespace CodexPlugin apiChecker = new ApiChecker(pluginTools); } - public RunningContainers[] BringOnline(CodexSetup codexSetup) + public RunningPod[] BringOnline(CodexSetup codexSetup) { LogSeparator(); Log($"Starting {codexSetup.Describe()}..."); @@ -34,14 +34,14 @@ namespace CodexPlugin { var podInfo = GetPodInfo(rc); var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}")); - Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})"); + Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})"); } LogSeparator(); return containers; } - public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers) + public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers) { var codexNodeFactory = new CodexNodeFactory(pluginTools); @@ -65,6 +65,14 @@ namespace CodexPlugin Log("Stopped."); } + public void Stop(RunningPod pod, bool waitTillStopped) + { + Log($"Stopping node..."); + var workflow = pluginTools.CreateWorkflow(); + workflow.Stop(pod, waitTillStopped); + Log("Stopped."); + } + public string GetCodexId() { if (versionResponse != null) return versionResponse.Version; @@ -85,7 +93,7 @@ namespace CodexPlugin return startupConfig; } - private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) + private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) { var futureContainers = new List(); for (var i = 0; i < numberOfNodes; i++) @@ -99,13 +107,13 @@ namespace CodexPlugin .ToArray(); } - private PodInfo GetPodInfo(RunningContainers rc) + private PodInfo GetPodInfo(RunningPod rc) { var workflow = pluginTools.CreateWorkflow(); return workflow.GetPodInfo(rc); } - private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory) + private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory) { var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory); @@ -122,10 +130,10 @@ namespace CodexPlugin return group; } - private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers) + private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers) { Log("Codex nodes failed to start"); - foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container); + foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container); } private void LogSeparator() diff --git a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs index 529b7fd..5f6065d 100644 --- a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs @@ -5,12 +5,12 @@ namespace CodexPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers[] DeployCodexNodes(this CoreInterface ci, int number, Action setup) + public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action setup) { return Plugin(ci).DeployCodexNodes(number, setup); } - public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers) + public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers) { return Plugin(ci).WrapCodexContainers(ci, containers); } diff --git a/ProjectPlugins/GethPlugin/GethDeployment.cs b/ProjectPlugins/GethPlugin/GethDeployment.cs index 8d7f0dd..e463ce4 100644 --- a/ProjectPlugins/GethPlugin/GethDeployment.cs +++ b/ProjectPlugins/GethPlugin/GethDeployment.cs @@ -7,9 +7,9 @@ namespace GethPlugin { public class GethDeployment : IHasContainer { - public GethDeployment(RunningContainers containers, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey) + public GethDeployment(RunningPod pod, Port discoveryPort, Port httpPort, Port wsPort, GethAccount account, string pubKey) { - Containers = containers; + Pod = pod; DiscoveryPort = discoveryPort; HttpPort = httpPort; WsPort = wsPort; @@ -17,9 +17,9 @@ namespace GethPlugin PubKey = pubKey; } - public RunningContainers Containers { get; } + public RunningPod Pod { get; } [JsonIgnore] - public RunningContainer Container { get { return Containers.Containers.Single(); } } + public RunningContainer Container { get { return Pod.Containers.Single(); } } public Port DiscoveryPort { get; } public Port HttpPort { get; } public Port WsPort { get; } diff --git a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs index 8392082..6d5859d 100644 --- a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs @@ -6,24 +6,24 @@ namespace MetricsPlugin { public static class CoreInterfaceExtensions { - public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets) + public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets) { return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray()); } - public static RunningContainers DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) + public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) { return Plugin(ci).DeployMetricsCollector(scrapeTargets); } - public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IHasMetricsScrapeTarget scrapeTarget) + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget) { - return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget); + return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget); } - public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget) + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget) { - return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget); + return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget); } public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets) diff --git a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs index b2c2b8e..bc7d926 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs @@ -31,15 +31,15 @@ namespace MetricsPlugin { } - public RunningContainers DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets) + public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets) { return starter.CollectMetricsFor(scrapeTargets); } - public IMetricsAccess WrapMetricsCollectorDeployment(RunningContainers runningContainer, IMetricsScrapeTarget target) + public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target) { - runningContainer = SerializeGate.Gate(runningContainer); - return starter.CreateAccessForTarget(runningContainer, target); + runningPod = SerializeGate.Gate(runningPod); + return starter.CreateAccessForTarget(runningPod, target); } public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName) diff --git a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs index f63f97d..159947a 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs @@ -16,7 +16,7 @@ namespace MetricsPlugin this.tools = tools; } - public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets) + public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets) { if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty."); @@ -32,9 +32,9 @@ namespace MetricsPlugin return runningContainers; } - public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target) + public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target) { - var metricsQuery = new MetricsQuery(tools, metricsContainer.Containers.Single()); + var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single()); return new MetricsAccess(metricsQuery, target); } diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs index 80203af..10c909c 100644 --- a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs +++ b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs @@ -49,8 +49,8 @@ namespace ContinuousTests var start = startUtc.ToString("o"); var end = endUtc.ToString("o"); - var containerName = container.RunningContainers.StartResult.Deployment.Name; - var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace; + var containerName = container.RunningPod.StartResult.Deployment.Name; + var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace; //container_name : codex3-5 - deploymentName as stored in pod // pod_namespace : codex - continuous - nolimits - tests - 1 diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 7c6c5a6..38bc502 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -125,8 +125,8 @@ namespace ContinuousTests foreach (var node in nodes) { var container = node.Container; - var deploymentName = container.RunningContainers.StartResult.Deployment.Name; - var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace; + var deploymentName = container.RunningPod.StartResult.Deployment.Name; + var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace; var openingLine = $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}"; elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, @@ -295,13 +295,13 @@ namespace ContinuousTests return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray(); } - private RunningContainers[] SelectRandomContainers() + private RunningPod[] SelectRandomContainers() { var number = handle.Test.RequiredNumberOfNodes; - var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList(); + var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList(); if (number == -1) return containers.ToArray(); - var result = new RunningContainers[number]; + var result = new RunningPod[number]; for (var i = 0; i < number; i++) { result[i] = containers.PickOneRandom(); diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index d3fb456..57c383c 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -43,13 +43,13 @@ namespace ContinuousTests var workflow = entryPoint.Tools.CreateWorkflow(); foreach (var instance in deployment.CodexInstances) { - foreach (var container in instance.Containers.Containers) + foreach (var container in instance.Pod.Containers) { var podInfo = workflow.GetPodInfo(container); log.Log($"Codex environment variables for '{container.Name}':"); log.Log( - $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " + - $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}"); + $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " + + $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}"); var codexVars = container.Recipe.EnvVars; foreach (var vars in codexVars) log.Log(vars.ToString()); log.Log(""); @@ -92,7 +92,7 @@ namespace ContinuousTests private void CheckCodexNodes(BaseLog log, Configuration config) { var nodes = entryPoint.CreateInterface() - .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray()); + .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray()); var pass = true; foreach (var n in nodes) { diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 299c4b9..e135d2f 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -132,7 +132,7 @@ namespace CodexTests.BasicTests private const string BytesStoredMetric = "codexRepostoreBytesUsed"; - private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainers rc) + private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningPod rc) { ScopedTestFiles(() => { @@ -154,7 +154,7 @@ namespace CodexTests.BasicTests var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value); return high > newBytes && newBytes > low; - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2)); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute)); FileUtils.TrackedFile? downloadedFile = null; LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId)); diff --git a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs index ccacdba..2604550 100644 --- a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs @@ -19,7 +19,7 @@ namespace CodexTests.BasicTests { node = Ci.StartCodexNode(); - Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait)); } [Test] @@ -27,7 +27,7 @@ namespace CodexTests.BasicTests { var myNode = Ci.StartCodexNode(); - Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects)); try { diff --git a/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs new file mode 100644 index 0000000..0e3fdaf --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/ClusterSpeedTests.cs @@ -0,0 +1,83 @@ +using DistTestCore; +using Logging; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class ClusterDiscSpeedTests : DistTest + { + private readonly Random random = new Random(); + + [Test] + [Combinatorial] + public void DiscSpeedTest( + [Values(1, 10, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024)] int bufferSizeKb + ) + { + long targetSize = (long)(1024 * 1024 * 1024) * 2; + long bufferSizeBytes = ((long)bufferSizeKb) * 1024; + + var filename = nameof(DiscSpeedTest); + + Thread.Sleep(2000); + if (File.Exists(filename)) File.Delete(filename); + Thread.Sleep(2000); + var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename); + Thread.Sleep(2000); + var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename); + + Log($"Write speed: {writeSpeed} per second."); + Log($"Read speed: {readSpeed} per second."); + } + + private ByteSize PerformWrite(long targetSize, long bufferSizeBytes, string filename) + { + long bytesWritten = 0; + var buffer = new byte[bufferSizeBytes]; + random.NextBytes(buffer); + + var sw = Stopwatch.Begin(GetTestLog()); + using (var stream = File.OpenWrite(filename)) + { + while (bytesWritten < targetSize) + { + long remaining = targetSize - bytesWritten; + long toWrite = Math.Min(bufferSizeBytes, remaining); + + stream.Write(buffer, 0, Convert.ToInt32(toWrite)); + bytesWritten += toWrite; + } + } + var duration = sw.End("WriteTime"); + double totalSeconds = duration.TotalSeconds; + double totalBytes = bytesWritten; + double bytesPerSecond = totalBytes / totalSeconds; + return new ByteSize(Convert.ToInt64(bytesPerSecond)); + } + + private ByteSize PerformRead(long targetSize, long bufferSizeBytes, string filename) + { + long bytesRead = 0; + var buffer = new byte[bufferSizeBytes]; + var sw = Stopwatch.Begin(GetTestLog()); + using (var stream = File.OpenRead(filename)) + { + while (bytesRead < targetSize) + { + long remaining = targetSize - bytesRead; + long toRead = Math.Min(bufferSizeBytes, remaining); + + var r = stream.Read(buffer, 0, Convert.ToInt32(toRead)); + bytesRead += r; + } + } + var duration = sw.End("ReadTime"); + double totalSeconds = duration.TotalSeconds; + double totalBytes = bytesRead; + double bytesPerSecond = totalBytes / totalSeconds; + return new ByteSize(Convert.ToInt64(bytesPerSecond)); + } + } +} diff --git a/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs new file mode 100644 index 0000000..f4c0a5f --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -0,0 +1,120 @@ +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class MultiPeerDownloadTests : AutoBootstrapDistTest + { + [Test] + [DontDownloadLogs] + [UseLongTimeouts] + [Combinatorial] + public void MultiPeerDownload( + [Values(5, 10, 20)] int numberOfHosts, + [Values(100, 1000)] int fileSize + ) + { + var hosts = AddCodex(numberOfHosts, s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var file = GenerateTestFile(fileSize.MB()); + var cid = hosts[0].UploadFile(file); + var tailOfManifestCid = cid.Id.Substring(cid.Id.Length - 6); + + var uploadLog = Ci.DownloadLog(hosts[0]); + var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block. + var blockCids = uploadLog + .FindLinesThatContain("Putting block into network store") + .Select(s => + { + var start = s.IndexOf("cid=") + 4; + var end = s.IndexOf(" count="); + var len = end - start; + return s.Substring(start, len); + }) + .ToArray(); + + Assert.That(blockCids.Length, Is.EqualTo(expectedNumberOfBlocks)); + + foreach (var h in hosts) h.DownloadContent(cid); + + var client = AddCodex(s => s.WithLogLevel(CodexPlugin.CodexLogLevel.Trace)); + var resultFile = client.DownloadContent(cid); + resultFile!.AssertIsEqual(file); + + var downloadLog = Ci.DownloadLog(client); + var host = string.Empty; + var blockCidHostMap = new Dictionary(); + downloadLog.IterateLines(line => + { + if (line.Contains("peer=") && line.Contains(" len=")) + { + var start = line.IndexOf("peer=") + 5; + var end = line.IndexOf(" len="); + var len = end - start; + host = line.Substring(start, len); + } + else if (!string.IsNullOrEmpty(host) && line.Contains("Storing block with key")) + { + var start = line.IndexOf("cid=") + 4; + var end = line.IndexOf(" count="); + var len = end - start; + var blockCid = line.Substring(start, len); + + blockCidHostMap.Add(blockCid, host); + host = string.Empty; + } + }); + + var totalFetched = blockCidHostMap.Count(p => !string.IsNullOrEmpty(p.Value)); + //PrintFullMap(blockCidHostMap); + PrintOverview(blockCidHostMap); + + Log("Expected number of blocks: " + expectedNumberOfBlocks); + Log("Total number of block CIDs found in dataset + manifest block: " + blockCids.Length); + Log("Total blocks fetched by hosts: " + totalFetched); + Assert.That(totalFetched, Is.EqualTo(expectedNumberOfBlocks)); + } + + private void PrintOverview(Dictionary blockCidHostMap) + { + var overview = new Dictionary(); + foreach (var pair in blockCidHostMap) + { + if (!overview.ContainsKey(pair.Value)) overview.Add(pair.Value, 1); + else overview[pair.Value]++; + } + + Log("Blocks fetched per host:"); + foreach (var pair in overview) + { + Log($"Host: {pair.Key} = {pair.Value}"); + } + } + + private void PrintFullMap(Dictionary blockCidHostMap) + { + Log("Per block, host it was fetched from:"); + foreach (var pair in blockCidHostMap) + { + if (string.IsNullOrEmpty(pair.Value)) + { + Log($"block: {pair.Key} = Not seen"); + } + else + { + Log($"block: {pair.Key} = '{pair.Value}'"); + } + } + } + + private long RoundUp(long filesize, long blockSize) + { + double f = filesize; + double b = blockSize; + + var result = Math.Ceiling(f / b); + return Convert.ToInt64(result); + } + } +} diff --git a/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs new file mode 100644 index 0000000..75dccbe --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/OneClientLargeFileTests.cs @@ -0,0 +1,39 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests +{ + [TestFixture] + public class OneClientLargeFileTests : CodexDistTest + { + [Test] + [Combinatorial] + [UseLongTimeouts] + public void OneClientLargeFile([Values( + 256, + 512, + 1024, // GB + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072 + )] int sizeMb) + { + var testFile = GenerateTestFile(sizeMb.MB()); + + var node = AddCodex(s => s + .WithLogLevel(CodexLogLevel.Warn) + .WithStorageQuota((sizeMb + 10).MB()) + ); + var contentId = node.UploadFile(testFile); + var downloadedFile = node.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs new file mode 100644 index 0000000..598f0bf --- /dev/null +++ b/Tests/CodexTests/ScalabilityTests/ScalabilityTests.cs @@ -0,0 +1,129 @@ +using CodexPlugin; +using DistTestCore; +using FileUtils; +using NUnit.Framework; +using Utils; + +namespace CodexTests.ScalabilityTests; + +[TestFixture] +public class ScalabilityTests : CodexDistTest +{ + private const string PatchedImage = "codexstorage/nim-codex:sha-9aeac06-dist-tests"; + private const string MasterImage = "codexstorage/nim-codex:sha-5380912-dist-tests"; + + /// + /// We upload a file to node A, then download it with B. + /// Then we stop node A, and download again with node C. + /// + [Test] + [Combinatorial] + [UseLongTimeouts] + [DontDownloadLogs] + public void ShouldMaintainFileInNetwork( + [Values(10, 40, 80, 100)] int numberOfNodes, + [Values(100, 1000, 5000, 10000)] int fileSizeInMb, + [Values(true, false)] bool usePatchedImage + ) + { + CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; + + var logLevel = CodexLogLevel.Info; + + var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); + var nodes = AddCodex(numberOfNodes - 1, s => s + .WithBootstrapNode(bootstrap) + .WithLogLevel(logLevel) + .WithStorageQuota((fileSizeInMb + 50).MB()) + ).ToList(); + + var uploader = nodes.PickOneRandom(); + var downloader = nodes.PickOneRandom(); + + var testFile = GenerateTestFile(fileSizeInMb.MB()); + var contentId = uploader.UploadFile(testFile); + var downloadedFile = downloader.DownloadContent(contentId); + + downloadedFile!.AssertIsEqual(testFile); + + uploader.Stop(true); + + var otherDownloader = nodes.PickOneRandom(); + downloadedFile = otherDownloader.DownloadContent(contentId); + + downloadedFile!.AssertIsEqual(testFile); + } + + /// + /// We upload a file to each node, to put a more wide-spread load on the network. + /// Then we run the same test as ShouldMaintainFileInNetwork. + /// + [Ignore("Make ShouldMaintainFileInNetwork pass reliably first.")] + [Test] + [Combinatorial] + [UseLongTimeouts] + [DontDownloadLogs] + public void EveryoneGetsAFile( + [Values(10, 40, 80, 100)] int numberOfNodes, + [Values(100, 1000)] int fileSizeInMb, + [Values(true, false)] bool usePatchedImage + ) + { + CodexContainerRecipe.DockerImageOverride = usePatchedImage ? PatchedImage : MasterImage; + + var logLevel = CodexLogLevel.Info; + + var bootstrap = AddCodex(s => s.WithLogLevel(logLevel)); + var nodes = AddCodex(numberOfNodes - 1, s => s + .WithBootstrapNode(bootstrap) + .WithLogLevel(logLevel) + .WithStorageQuota((fileSizeInMb + 50).MB()) + ).ToList(); + + var pairTasks = nodes.Select(n => + { + return Task.Run(() => + { + var file = GenerateTestFile(fileSizeInMb.MB()); + var cid = n.UploadFile(file); + return new NodeFilePair(n, file, cid); + }); + }); + + var pairs = pairTasks.Select(t => Time.Wait(t)).ToList(); + + RunDoubleDownloadTest( + pairs.PickOneRandom(), + pairs.PickOneRandom(), + pairs.PickOneRandom() + ); + } + + private void RunDoubleDownloadTest(NodeFilePair source, NodeFilePair dl1, NodeFilePair dl2) + { + var expectedFile = source.File; + var cid = source.Cid; + + var file1 = dl1.Node.DownloadContent(cid); + file1!.AssertIsEqual(expectedFile); + + source.Node.Stop(true); + + var file2 = dl2.Node.DownloadContent(cid); + file2!.AssertIsEqual(expectedFile); + } + + public class NodeFilePair + { + public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid) + { + Node = node; + File = file; + Cid = cid; + } + + public ICodexNode Node { get; } + public TrackedFile File { get; } + public ContentId Cid { get; } + } +} diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index b1a94da..3fa34b5 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -24,6 +24,9 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; } + /// + /// Does not override [DontDownloadLogs] attribute. + /// public bool AlwaysDownloadContainerLogs { get; set; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace) @@ -36,7 +39,7 @@ namespace DistTestCore var config = new KubernetesWorkflow.Configuration( kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay(), + retryDelay: timeSet.K8sOperationRetryDelay(), kubernetesNamespace: k8sNamespace ); diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 76f9d5b..ed99fe9 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -98,7 +98,7 @@ namespace DistTestCore } catch (Exception ex) { - fixtureLog.Error("Cleanup failed: " + ex.Message); + fixtureLog.Error("Cleanup failed: " + ex); GlobalTestFailure.HasFailed = true; } } @@ -236,9 +236,19 @@ namespace DistTestCore } private bool ShouldUseLongTimeouts() + { + return CurrentTestMethodHasAttribute(); + } + + private bool HasDontDownloadAttribute() + { + return CurrentTestMethodHasAttribute(); + } + + private bool CurrentTestMethodHasAttribute() where T : PropertyAttribute { // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test. - // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out + // But this doesn't work for tests making use of [TestCase] or [Combinatorial]. So instead, we use reflection here to figure out // if the attribute is present. var currentTest = TestContext.CurrentContext.Test; var className = currentTest.ClassName; @@ -247,7 +257,7 @@ namespace DistTestCore var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); - return testMethods.Any(m => m.GetCustomAttribute() != null); + return testMethods.Any(m => m.GetCustomAttribute() != null); } private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) @@ -268,9 +278,10 @@ namespace DistTestCore private bool ShouldDownloadAllLogs(TestStatus testStatus) { if (configuration.AlwaysDownloadContainerLogs) return true; + if (!IsDownloadingLogsEnabled()) return false; if (testStatus == TestStatus.Failed) { - return IsDownloadingLogsEnabled(); + return true; } return false; @@ -288,8 +299,7 @@ namespace DistTestCore private bool IsDownloadingLogsEnabled() { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey); + return !HasDontDownloadAttribute(); } } diff --git a/Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs b/Tests/DistTestCore/DontDownloadLogsAttribute.cs similarity index 67% rename from Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs rename to Tests/DistTestCore/DontDownloadLogsAttribute.cs index 800b35b..13baab8 100644 --- a/Tests/DistTestCore/DontDownloadLogsOnFailureAttribute.cs +++ b/Tests/DistTestCore/DontDownloadLogsAttribute.cs @@ -3,11 +3,11 @@ namespace DistTestCore { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsOnFailureAttribute : PropertyAttribute + public class DontDownloadLogsAttribute : PropertyAttribute { public const string DontDownloadKey = "DontDownloadLogs"; - public DontDownloadLogsOnFailureAttribute() + public DontDownloadLogsAttribute() : base(DontDownloadKey) { } diff --git a/Tests/DistTestCore/Helpers/AssertHelpers.cs b/Tests/DistTestCore/Helpers/AssertHelpers.cs index efd0749..c0f995d 100644 --- a/Tests/DistTestCore/Helpers/AssertHelpers.cs +++ b/Tests/DistTestCore/Helpers/AssertHelpers.cs @@ -14,7 +14,7 @@ namespace DistTestCore.Helpers Time.WaitUntil(() => { var c = constraint.Resolve(); return c.ApplyTo(actual()).IsSuccess; - }); + }, "RetryAssert: " + message); } catch (TimeoutException) { diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 50cb5fb..542ca27 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -13,7 +13,7 @@ namespace DistTestCore private const string TestsType = "dist-tests"; private readonly EntryPoint entryPoint; private readonly Dictionary metadata; - private readonly List runningContainers = new(); + private readonly List runningContainers = new(); private readonly string deployId; public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId) @@ -65,12 +65,12 @@ namespace DistTestCore return DateTime.UtcNow - TestStart; } - public void OnContainersStarted(RunningContainers rc) + public void OnContainersStarted(RunningPod rc) { runningContainers.Add(rc); } - public void OnContainersStopped(RunningContainers rc) + public void OnContainersStopped(RunningPod rc) { runningContainers.Remove(rc); } @@ -93,13 +93,20 @@ namespace DistTestCore public void DownloadAllLogs() { - foreach (var rc in runningContainers) + try { - foreach (var c in rc.Containers) + foreach (var rc in runningContainers) { - CoreInterface.DownloadLog(c); + foreach (var c in rc.Containers) + { + CoreInterface.DownloadLog(c); + } } } + catch (Exception ex) + { + Log.Error("Exception during log download: " + ex); + } } } } diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 440d092..adea96d 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -122,7 +122,7 @@ namespace CodexNetDeployer }); } - private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, + private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, CodexContractsDeployment contractsDeployment) { if (!config.DeployDiscordBot) return null; @@ -155,7 +155,7 @@ namespace CodexNetDeployer return rc; } - private RunningContainers? StartMetricsService(CoreInterface ci, List startResults) + private RunningPod? StartMetricsService(CoreInterface ci, List startResults) { if (!config.MetricsScraper || !startResults.Any()) return null; @@ -180,7 +180,7 @@ namespace CodexNetDeployer private CodexInstance CreateCodexInstance(ICodexNode node) { - return new CodexInstance(node.Container.RunningContainers, node.GetDebugInfo()); + return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo()); } private string? GetKubeConfig(string kubeConfigFile) @@ -270,7 +270,7 @@ namespace CodexNetDeployer return TimeSpan.FromMinutes(10); } - public TimeSpan WaitForK8sServiceDelay() + public TimeSpan K8sOperationRetryDelay() { return TimeSpan.FromSeconds(30); } diff --git a/Tools/CodexNetDeployer/K8sHook.cs b/Tools/CodexNetDeployer/K8sHook.cs index b4c0c16..b8c78b7 100644 --- a/Tools/CodexNetDeployer/K8sHook.cs +++ b/Tools/CodexNetDeployer/K8sHook.cs @@ -18,11 +18,11 @@ namespace CodexNetDeployer this.metadata = metadata; } - public void OnContainersStarted(RunningContainers rc) + public void OnContainersStarted(RunningPod rc) { } - public void OnContainersStopped(RunningContainers rc) + public void OnContainersStopped(RunningPod rc) { }