From fff02656b72890cc4055b66d7bf8a620b3ec9d52 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 11 Aug 2023 08:39:51 +0200 Subject: [PATCH] automatically determine runner location --- CodexNetDeployer/Configuration.cs | 3 -- CodexNetDeployer/Deployer.cs | 1 - CodexNetDeployer/Program.cs | 6 --- CodexNetDownloader/Configuration.cs | 3 -- CodexNetDownloader/Program.cs | 7 +-- ContinuousTests/CodexAccessFactory.cs | 2 +- ContinuousTests/Configuration.cs | 8 +-- ContinuousTests/ContinuousTestRunner.cs | 4 +- ContinuousTests/K8sFactory.cs | 3 +- ContinuousTests/NodeRunner.cs | 2 +- ContinuousTests/SingleTestRun.cs | 4 +- DistTestCore/Configuration.cs | 67 +++++++++++++++++++++---- 12 files changed, 67 insertions(+), 43 deletions(-) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index dbcc8bf9..ee7ac7e1 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,5 +1,4 @@ using ArgsUniform; -using DistTestCore; using DistTestCore.Codex; namespace CodexNetDeployer @@ -51,8 +50,6 @@ namespace CodexNetDeployer "set this option to override the label value.")] public string TestsTypePodLabel { get; set; } = "continuous-tests"; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; - public List Validate() { var errors = new List(); diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index a5e174e4..feb750ed 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -68,7 +68,6 @@ namespace CodexNetDeployer logDebug: false, dataFilesPath: "notUsed", codexLogLevel: config.CodexLogLevel, - runnerLocation: config.RunnerLocation, k8sNamespacePrefix: config.KubeNamespace ); diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index d96b25f1..9c1ecbf7 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,6 +1,5 @@ using ArgsUniform; using CodexNetDeployer; -using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; using DistTestCore.Metrics; @@ -17,11 +16,6 @@ public class Program var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); - if (args.Any(a => a == "--external")) - { - config.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } - var errors = config.Validate(); if (errors.Any()) { diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs index 008dfbf9..a42136a4 100644 --- a/CodexNetDownloader/Configuration.cs +++ b/CodexNetDownloader/Configuration.cs @@ -1,5 +1,4 @@ using ArgsUniform; -using DistTestCore; using DistTestCore.Codex; namespace CodexNetDownloader @@ -16,7 +15,5 @@ namespace CodexNetDownloader public string KubeConfigFile { get; set; } = "null"; public CodexDeployment CodexDeployment { get; set; } = null!; - - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; } } diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index 195e96be..5744e71e 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -15,17 +15,12 @@ public class Program var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); - if (args.Any(a => a == "--external")) - { - config.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } - config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson); if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/ContinuousTests/CodexAccessFactory.cs b/ContinuousTests/CodexAccessFactory.cs index a149017c..78e90693 100644 --- a/ContinuousTests/CodexAccessFactory.cs +++ b/ContinuousTests/CodexAccessFactory.cs @@ -12,7 +12,7 @@ namespace ContinuousTests return containers.Select(container => { var address = container.ClusterExternalAddress; - if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress; + if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress; return new CodexAccess(log, container, timeSet, address); }).ToArray(); } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index b0cebbd6..da1eaeb7 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -30,7 +30,7 @@ namespace ContinuousTests public CodexDeployment CodexDeployment { get; set; } = null!; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; + public RunnerLocation RunnerLocation { get; set; } } public class ConfigLoader @@ -42,10 +42,7 @@ namespace ContinuousTests var result = uniformArgs.Parse(true); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); - if (args.Any(a => a == "--external")) - { - result.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } + result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First()); return result; } @@ -63,7 +60,6 @@ namespace ContinuousTests Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl + "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl); - Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 35ff486a..533d146b 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -60,7 +60,7 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log); lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources(); } @@ -71,7 +71,7 @@ namespace ContinuousTests var path = Path.Combine(config.LogPath, "containers"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); - var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); taskFactory.Run(downloader.Run); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index ff9b7e93..6ac661eb 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -6,7 +6,7 @@ namespace ContinuousTests { public class K8sFactory { - public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) + public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -16,7 +16,6 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: runnerLocation, k8sNamespacePrefix: customNamespace ); diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 4bb335e6..5b79bf5c 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -91,7 +91,7 @@ namespace ContinuousTests private TestLifecycle CreateTestLifecycle() { - return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); + return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index cff325eb..bbc4888c 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -138,7 +138,7 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -221,7 +221,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, config.RunnerLocation, string.Empty); + CodexLogLevel.Error, string.Empty); } } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index daccad65..23dcdb15 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using KubernetesWorkflow; +using System.Net.NetworkInformation; using Utils; namespace DistTestCore @@ -11,7 +12,6 @@ namespace DistTestCore private readonly bool logDebug; private readonly string dataFilesPath; private readonly CodexLogLevel codexLogLevel; - private readonly TestRunnerLocation runnerLocation; private readonly string k8sNamespacePrefix; public Configuration() @@ -21,18 +21,16 @@ namespace DistTestCore logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); - runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); k8sNamespacePrefix = "ct-"; } - public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix) + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix) { this.kubeConfigFile = kubeConfigFile; this.logPath = logPath; this.logDebug = logDebug; this.dataFilesPath = dataFilesPath; this.codexLogLevel = codexLogLevel; - this.runnerLocation = runnerLocation; this.k8sNamespacePrefix = k8sNamespacePrefix; } @@ -61,14 +59,16 @@ namespace DistTestCore return codexLogLevel; } - public TestRunnerLocation GetTestRunnerLocation() - { - return runnerLocation; - } + private RunnerLocation? runnerLocation = null; public Address GetAddress(RunningContainer container) { - if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster) + if (runnerLocation == null) + { + runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container); + } + + if (runnerLocation == RunnerLocation.InternalToCluster) { return container.ClusterInternalAddress; } @@ -90,9 +90,56 @@ namespace DistTestCore } } - public enum TestRunnerLocation + public enum RunnerLocation { ExternalToCluster, InternalToCluster, } + + public class RunnerLocationUtils + { + private static bool alreadyDidThat = false; + + public static RunnerLocation DetermineRunnerLocation(RunningContainer container) + { + // We want to be sure we don't ping more often than strictly necessary. + // If we have already determined the location during this application + // lifetime, don't do it again. + if (alreadyDidThat) throw new Exception("We already did that."); + alreadyDidThat = true; + + if (PingHost(container.ClusterInternalAddress)) + { + return RunnerLocation.InternalToCluster; + } + if (PingHost(container.ClusterExternalAddress)) + { + return RunnerLocation.ExternalToCluster; + } + + throw new Exception("Unable to determine runner location."); + } + + private static string Format(Address host) + { + return host.Host + .Replace("http://", "") + .Replace("https://", ""); + } + + private static bool PingHost(Address host) + { + try + { + using var pinger = new Ping(); + PingReply reply = pinger.Send(Format(host)); + return reply.Status == IPStatus.Success; + } + catch (PingException) + { + } + + return false; + } + } }