diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index c43c982..a290193 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -17,8 +17,8 @@ namespace CodexDistTestCore.Config public KubernetesClientConfiguration GetK8sClientConfig() { if (config != null) return config; - config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - //config = KubernetesClientConfiguration.BuildDefaultConfig(); + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); return config; } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index f86cdbf..f9dace8 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -58,9 +58,9 @@ K8s(k => k.FetchPodLog(node, logHandler)); } - public PrometheusInfo BringOnlinePrometheus(string config) + public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) { - var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config); + var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); PrometheusInfo? info = null; K8s(k => info = k.BringOnlinePrometheus(spec)); diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs index 1dc8c28..dcda941 100644 --- a/CodexDistTestCore/K8sPrometheusSpecs.cs +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -11,17 +11,19 @@ namespace CodexDistTestCore private const string portName = "prom-1"; private readonly string config; - public K8sPrometheusSpecs(int servicePort, string config) + public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) { ServicePort = servicePort; + PrometheusNumber = prometheusNumber; this.config = config; } public int ServicePort { get; } + public int PrometheusNumber { get; } public string GetDeploymentName() { - return "test-prom"; + return "test-prom" + PrometheusNumber; } public V1Deployment CreatePrometheusDeployment() @@ -88,7 +90,7 @@ namespace CodexDistTestCore ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = "codex-prom-service", + Name = "codex-prom-service" + PrometheusNumber, NamespaceProperty = K8sCluster.K8sNamespace }, Spec = new V1ServiceSpec @@ -99,7 +101,7 @@ namespace CodexDistTestCore { new V1ServicePort { - Name = "prom-service", + Name = "prom-service" + PrometheusNumber, Protocol = "TCP", Port = 9090, TargetPort = portName, diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 70a6ccb..ca10fc3 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using NUnit.Framework; namespace CodexDistTestCore { @@ -11,18 +12,23 @@ namespace CodexDistTestCore { private readonly K8sCluster k8sCluster = new K8sCluster(); private readonly Http http; + private readonly OnlineCodexNode[] nodes; - public MetricsAccess(PrometheusInfo prometheusInfo) + public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes) { http = new Http( k8sCluster.GetIp(), prometheusInfo.ServicePort, "api/v1"); + this.nodes = nodes; } public int? GetMostRecentInt(string metricName, IOnlineCodexNode node) { var n = (OnlineCodexNode)node; + CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + + "(This CodexNode is tracked by a different instance.)"); + var pod = n.Group.PodInfo!; var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 594d311..2e4696b 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -5,9 +5,10 @@ namespace CodexDistTestCore { public class MetricsAggregator { + private readonly NumberSource prometheusNumberSource = new NumberSource(0); private readonly TestLog log; private readonly K8sManager k8sManager; - private PrometheusInfo? activePrometheus; + private readonly Dictionary activePrometheuses = new Dictionary(); public MetricsAggregator(TestLog log, K8sManager k8sManager) { @@ -17,31 +18,27 @@ namespace CodexDistTestCore public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes) { - if (activePrometheus != null) + var alreadyStartedNodes = nodes.Where(n => activePrometheuses.Values.Any(v => v.Contains(n))); + if (alreadyStartedNodes.Any()) { - Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :("); + Assert.Fail("Incorrect test setup: 'GatherMetrics' was already called on one or more of these OnlineCodexNodes."); throw new InvalidOperationException(); } log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); var config = GeneratePrometheusConfig(nodes); - StartPrometheusPod(config); + var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); + activePrometheuses.Add(prometheus, nodes); log.Log("Metrics service started."); - return new MetricsAccess(activePrometheus!); + return new MetricsAccess(prometheus, nodes); } public void DownloadAllMetrics() { } - private void StartPrometheusPod(string config) - { - if (activePrometheus != null) return; - activePrometheus = k8sManager.BringOnlinePrometheus(config); - } - private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) { var config = ""; diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 74ec0cb..e0dd351 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -39,14 +39,29 @@ namespace Tests.BasicTests var metrics = GatherMetrics(group); + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); + + var metrics2 = GatherMetrics(group2); + var primary = group[0]; var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); AssertWithTimeout( () => metrics.GetMostRecentInt("libp2p_peers", primary), isEqualTo: 1, "Number of peers metric was incorrect."); + + AssertWithTimeout( + () => metrics2.GetMostRecentInt("libp2p_peers", primary2), + isEqualTo: 1, + "Aaa"); } [Test] @@ -84,19 +99,19 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - [Test] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; + //[Test] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) {