Allows starting of multiple metrics-collecting services

CodexDistTestCore/Config/K8sCluster.cs

@@ -17,8 +17,8 @@ namespace CodexDistTestCore.Config
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
             if (config != null) return config;
-            config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
+            //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
-            //config = KubernetesClientConfiguration.BuildDefaultConfig();
+            config = KubernetesClientConfiguration.BuildDefaultConfig();
             return config;
         }
 

CodexDistTestCore/K8sManager.cs

@@ -58,9 +58,9 @@
             K8s(k => k.FetchPodLog(node, logHandler));
         }
 
-        public PrometheusInfo BringOnlinePrometheus(string config)
+        public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
         {
-            var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config);
+            var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
 
             PrometheusInfo? info = null;
             K8s(k => info = k.BringOnlinePrometheus(spec));

CodexDistTestCore/K8sPrometheusSpecs.cs

@@ -11,17 +11,19 @@ namespace CodexDistTestCore
         private const string portName = "prom-1";
         private readonly string config;
 
-        public K8sPrometheusSpecs(int servicePort, string config)
+        public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
         {
             ServicePort = servicePort;
+            PrometheusNumber = prometheusNumber;
             this.config = config;
         }
 
         public int ServicePort { get; }
+        public int PrometheusNumber { get; }
 
         public string GetDeploymentName()
         {
-            return "test-prom";
+            return "test-prom" + PrometheusNumber;
         }
 
         public V1Deployment CreatePrometheusDeployment()
@@ -88,7 +90,7 @@ namespace CodexDistTestCore
                 ApiVersion = "v1",
                 Metadata = new V1ObjectMeta
                 {
-                    Name = "codex-prom-service",
+                    Name = "codex-prom-service" + PrometheusNumber,
                     NamespaceProperty = K8sCluster.K8sNamespace
                 },
                 Spec = new V1ServiceSpec
@@ -99,7 +101,7 @@ namespace CodexDistTestCore
                     {
                         new V1ServicePort
                         {
-                            Name = "prom-service",
+                            Name = "prom-service" + PrometheusNumber,
                             Protocol = "TCP",
                             Port = 9090,
                             TargetPort = portName,

CodexDistTestCore/MetricsAccess.cs

@@ -1,4 +1,5 @@
 using CodexDistTestCore.Config;
+using NUnit.Framework;
 
 namespace CodexDistTestCore
 {
@@ -11,18 +12,23 @@ namespace CodexDistTestCore
     {
         private readonly K8sCluster k8sCluster = new K8sCluster();
         private readonly Http http;
+        private readonly OnlineCodexNode[] nodes;
 
-        public MetricsAccess(PrometheusInfo prometheusInfo)
+        public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes)
         {
             http = new Http(
                 k8sCluster.GetIp(),
                 prometheusInfo.ServicePort,
                 "api/v1");
+            this.nodes = nodes;
         }
 
         public int? GetMostRecentInt(string metricName, IOnlineCodexNode node)
         {
             var n = (OnlineCodexNode)node;
+            CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
+                "(This CodexNode is tracked by a different instance.)");
+
             var pod = n.Group.PodInfo!;
 
             var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}[12h])");

CodexDistTestCore/MetricsAggregator.cs

@@ -5,9 +5,10 @@ namespace CodexDistTestCore
 {
     public class MetricsAggregator
     {
+        private readonly NumberSource prometheusNumberSource = new NumberSource(0);
         private readonly TestLog log;
         private readonly K8sManager k8sManager;
-        private PrometheusInfo? activePrometheus;
+        private readonly Dictionary<PrometheusInfo, OnlineCodexNode[]> activePrometheuses = new Dictionary<PrometheusInfo, OnlineCodexNode[]>();
 
         public MetricsAggregator(TestLog log, K8sManager k8sManager)
         {
@@ -17,31 +18,27 @@ namespace CodexDistTestCore
 
         public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
         {
-            if (activePrometheus != null)
+            var alreadyStartedNodes = nodes.Where(n => activePrometheuses.Values.Any(v => v.Contains(n)));
+            if (alreadyStartedNodes.Any())
             {
-                Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :(");
+                Assert.Fail("Incorrect test setup: 'GatherMetrics' was already called on one or more of these OnlineCodexNodes.");
                 throw new InvalidOperationException();
             }
 
             log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
 
             var config = GeneratePrometheusConfig(nodes);
-            StartPrometheusPod(config);
+            var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
+            activePrometheuses.Add(prometheus, nodes);
 
             log.Log("Metrics service started.");
-            return new MetricsAccess(activePrometheus!);
+            return new MetricsAccess(prometheus, nodes);
         }
 
         public void DownloadAllMetrics()
         {
         }
 
-        private void StartPrometheusPod(string config)
-        {
-            if (activePrometheus != null) return;
-            activePrometheus = k8sManager.BringOnlinePrometheus(config);
-        }
-
         private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
         {
             var config = "";

Tests/BasicTests/SimpleTests.cs

@@ -39,14 +39,29 @@ namespace Tests.BasicTests
 
             var metrics = GatherMetrics(group);
 
+            var group2 = SetupCodexNodes(2)
+                .EnableMetrics()
+                .BringOnline();
+
+            var metrics2 = GatherMetrics(group2);
+
             var primary = group[0];
             var secondary = group[1];
+            var primary2 = group2[0];
+            var secondary2 = group2[1];
+
             primary.ConnectToPeer(secondary);
+            primary2.ConnectToPeer(secondary2);
 
             AssertWithTimeout(
                 () => metrics.GetMostRecentInt("libp2p_peers", primary),
                 isEqualTo: 1,
                 "Number of peers metric was incorrect.");
+
+            AssertWithTimeout(
+                () => metrics2.GetMostRecentInt("libp2p_peers", primary2),
+                isEqualTo: 1,
+                "Aaa");
         }
 
         [Test]
@@ -84,19 +99,19 @@ namespace Tests.BasicTests
             PerformTwoClientTest(primary, secondary);
         }
 
-        [Test]
+        //[Test]
-        public void TwoClientsTwoLocationsTest()
+        //public void TwoClientsTwoLocationsTest()
-        {
+        //{
-            var primary = SetupCodexNodes(1)
+        //    var primary = SetupCodexNodes(1)
-                            .At(Location.BensLaptop)
+        //                    .At(Location.BensLaptop)
-                            .BringOnline()[0];
+        //                    .BringOnline()[0];
 
-            var secondary = SetupCodexNodes(1)
+        //    var secondary = SetupCodexNodes(1)
-                            .At(Location.BensOldGamingMachine)
+        //                    .At(Location.BensOldGamingMachine)
-                            .BringOnline()[0];
+        //                    .BringOnline()[0];
 
-            PerformTwoClientTest(primary, secondary);
+        //    PerformTwoClientTest(primary, secondary);
-        }
+        //}
 
         private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
         {