Merge pull request #1 from benbierens/feature/metrics

Feature/metrics

CodexDistTestCore/CodexNodeContainer.cs

@@ -2,7 +2,7 @@
 {
     public class CodexNodeContainer
     {
-        public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir)
+        public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort)
         {
             Name = name;
             ServicePort = servicePort;
@@ -12,6 +12,7 @@
             DiscoveryPort = discoveryPort;
             ListenPort = listenPort;
             DataDir = dataDir;
+            MetricsPort = metricsPort;
         }
 
         public string Name { get; }
@@ -22,6 +23,7 @@
         public int DiscoveryPort { get; }
         public int ListenPort { get; }
         public string DataDir { get; }
+        public int MetricsPort { get; }
     }
 
     public class CodexGroupNumberSource
@@ -57,7 +59,7 @@
             this.groupContainerFactory = groupContainerFactory;
         }
 
-        public CodexNodeContainer CreateNext()
+        public CodexNodeContainer CreateNext(OfflineCodexNodes offline)
         {
             var n = containerNameSource.GetNextNumber();
             return new CodexNodeContainer(
@@ -68,8 +70,15 @@
                 containerPortName: $"api-{n}",
                 discoveryPort: codexPortSource.GetNextNumber(),
                 listenPort: codexPortSource.GetNextNumber(),
-                dataDir: $"datadir{n}"
+                dataDir: $"datadir{n}",
+                metricsPort: GetMetricsPort(offline)
             );
         }
+
+        private int GetMetricsPort(OfflineCodexNodes offline)
+        {
+            if (offline.MetricsEnabled) return codexPortSource.GetNextNumber();
+            return 0;
+        }
     }
 }

CodexDistTestCore/CodexNodeLog.cs

@@ -2,7 +2,12 @@
 
 namespace CodexDistTestCore
 {
-    public class CodexNodeLog
+    public interface ICodexNodeLog
+    {
+        void AssertLogContains(string expectedString);
+    }
+
+    public class CodexNodeLog : ICodexNodeLog
     {
         private readonly LogFile logFile;
 

CodexDistTestCore/Config/CodexDockerImage.cs

@@ -25,12 +25,12 @@ namespace CodexDistTestCore.Config
         {
             public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
 
-            public void Create(OfflineCodexNodes node, CodexNodeContainer environment)
+            public void Create(OfflineCodexNodes node, CodexNodeContainer container)
             {
-                AddVar("API_PORT", environment.ApiPort.ToString());
+                AddVar("API_PORT", container.ApiPort.ToString());
-                AddVar("DATA_DIR", environment.DataDir);
+                AddVar("DATA_DIR", container.DataDir);
-                AddVar("DISC_PORT", environment.DiscoveryPort.ToString());
+                AddVar("DISC_PORT", container.DiscoveryPort.ToString());
-                AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}");
+                AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}");
 
                 if (node.BootstrapNode != null)
                 {
@@ -45,6 +45,11 @@ namespace CodexDistTestCore.Config
                 {
                     AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!);
                 }
+                if (node.MetricsEnabled)
+                {
+                    AddVar("METRICS_ADDR", "0.0.0.0");
+                    AddVar("METRICS_PORT", container.MetricsPort.ToString());
+                }
             }
 
             private void AddVar(string key, string value)

CodexDistTestCore/Config/K8sCluster.cs

@@ -17,7 +17,8 @@ namespace CodexDistTestCore.Config
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
             if (config != null) return config;
-            config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
+            //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
+            config = KubernetesClientConfiguration.BuildDefaultConfig();
             return config;
         }
 

CodexDistTestCore/DistTest.cs

@@ -57,7 +57,7 @@ namespace CodexDistTestCore
             try
             {
                 log.EndTest();
-                IncludeLogsOnTestFailure();
+                IncludeLogsAndMetricsOnTestFailure();
                 k8sManager.DeleteAllResources();
                 fileManager.DeleteAllTestFiles();
             }
@@ -78,19 +78,20 @@ namespace CodexDistTestCore
             return new OfflineCodexNodes(k8sManager, numberOfNodes);
         }
 
-        private void IncludeLogsOnTestFailure()
+        private void IncludeLogsAndMetricsOnTestFailure()
         {
             var result = TestContext.CurrentContext.Result;
             if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
             {
-                if (IsDownloadingLogsEnabled())
+                if (IsDownloadingLogsAndMetricsEnabled())
                 {
-                    log.Log("Downloading all CodexNode logs because of test failure...");
+                    log.Log("Downloading all CodexNode logs and metrics because of test failure...");
                     k8sManager.ForEachOnlineGroup(DownloadLogs);
+                    k8sManager.DownloadAllMetrics();
                 }
                 else
                 {
-                    log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute.");
+                    log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
                 }
             }
         }
@@ -105,7 +106,7 @@ namespace CodexDistTestCore
             }
         }
 
-        private bool IsDownloadingLogsEnabled()
+        private bool IsDownloadingLogsAndMetricsEnabled()
         {
             var testProperties = TestContext.CurrentContext.Test.Properties;
             return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);

CodexDistTestCore/FileManager.cs

@@ -80,28 +80,30 @@ namespace CodexDistTestCore
             return info.Length;
         }
 
-        public void AssertIsEqual(TestFile? other)
+        public void AssertIsEqual(TestFile? actual)
         {
-            if (other == null) Assert.Fail("TestFile is null.");
+            if (actual == null) Assert.Fail("TestFile is null.");
-            if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
+            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
 
-            using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
-            using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);
 
-            var bytes1 = new byte[FileManager.ChunkSize];
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
-            var bytes2 = new byte[FileManager.ChunkSize];
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
 
-            var read1 = 0;
+            var bytesExpected = new byte[FileManager.ChunkSize];
-            var read2 = 0;
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
 
             while (true)
             {
-                read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize);
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
-                read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
 
-                if (read1 == 0 && read2 == 0) return;
+                if (readExpected == 0 && readActual == 0) return;
-                Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length.");
+                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
-                CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal.");
+                CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
             }
         }
     }

CodexDistTestCore/K8sManager.cs

@@ -14,11 +14,13 @@
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly TestLog log;
         private readonly IFileManager fileManager;
+        private readonly MetricsAggregator metricsAggregator;
 
         public K8sManager(TestLog log, IFileManager fileManager)
         {
             this.log = log;
             this.fileManager = fileManager;
+            metricsAggregator = new MetricsAggregator(log, this);
         }
 
         public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
@@ -29,6 +31,11 @@
 
             log.Log($"{online.Describe()} online.");
 
+            if (offline.MetricsEnabled)
+            {
+                BringOnlineMetrics(online);
+            }
+
             return online;
         }
 
@@ -58,20 +65,41 @@
             K8s(k => k.FetchPodLog(node, logHandler));
         }
 
+        public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
+        {
+            var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
+
+            PrometheusInfo? info = null;
+            K8s(k => info = k.BringOnlinePrometheus(spec));
+            return info!;
+        }
+
+        public void DownloadAllMetrics()
+        {
+            metricsAggregator.DownloadAllMetrics();
+        }
+
+        private void BringOnlineMetrics(CodexNodeGroup group)
+        {
+            var onlineNodes = group.Nodes.Cast<OnlineCodexNode>().ToArray();
+
+            metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
+        }
+
         private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
         {
-            var containers = CreateContainers(offline.NumberOfNodes);
+            var containers = CreateContainers(offline);
             var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
             var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
             onlineCodexNodeGroups.Add(result);
             return result;
         }
 
-        private CodexNodeContainer[] CreateContainers(int number)
+        private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline)
         {
             var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
             var containers = new List<CodexNodeContainer>();
-            for (var i = 0; i < number; i++) containers.Add(factory.CreateNext());
+            for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline));
             return containers.ToArray();
         }
 

CodexDistTestCore/K8sOperations.cs

@@ -1,5 +1,6 @@
 using CodexDistTestCore.Config;
 using k8s;
+using k8s.KubeConfigModels;
 using k8s.Models;
 using NUnit.Framework;
 
@@ -57,7 +58,23 @@ namespace CodexDistTestCore
             logHandler.Log(stream);
         }
 
+        public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
+        {
+            EnsureTestNamespace();
+
+            CreatePrometheusDeployment(spec);
+            CreatePrometheusService(spec);
+            WaitUntilPrometheusOnline(spec);
+
+            return new PrometheusInfo(spec.ServicePort, FetchNewPod());
+        }
+
         private void FetchPodInfo(CodexNodeGroup online)
+        {
+            online.PodInfo = FetchNewPod();
+        }
+
+        private PodInfo FetchNewPod()
         {
             var pods = client.ListNamespacedPod(K8sNamespace).Items;
 
@@ -65,12 +82,13 @@ namespace CodexDistTestCore
             Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
 
             var newPod = newPods.Single();
-            online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP);
+            var info = new PodInfo(newPod.Name(), newPod.Status.PodIP);
 
-            Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure.");
+            Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure.");
-            Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure.");
+            Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure.");
 
             knownPods.Add(newPod.Name());
+            return info;
         }
 
         #region Waiting
@@ -103,6 +121,16 @@ namespace CodexDistTestCore
             WaitUntil(() => !IsTestNamespaceOnline());
         }
 
+        private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec)
+        {
+            var deploymentName = spec.GetDeploymentName();
+            WaitUntil(() =>
+            {
+                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
+                return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
+            });
+        }
+
         private void WaitUntil(Func<bool> predicate)
         {
             var start = DateTime.UtcNow;
@@ -166,6 +194,11 @@ namespace CodexDistTestCore
             online.Service = null;
         }
 
+        private void CreatePrometheusService(K8sPrometheusSpecs spec)
+        {
+            client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
+        }
+
         #endregion
 
         #region Deployment management
@@ -232,6 +265,7 @@ namespace CodexDistTestCore
                     Env = dockerImage.CreateEnvironmentVariables(offline, container)
                 });
             }
+
             return result;
         }
 
@@ -242,6 +276,11 @@ namespace CodexDistTestCore
             online.Deployment = null;
         }
 
+        private void CreatePrometheusDeployment(K8sPrometheusSpecs spec)
+        {
+            client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace);
+        }
+
         #endregion
 
         #region Namespace management

CodexDistTestCore/K8sPrometheusSpecs.cs

@@ -0,0 +1,122 @@
+using CodexDistTestCore.Config;
+using k8s.Models;
+
+namespace CodexDistTestCore
+{
+    public class K8sPrometheusSpecs
+    {
+        public const string ContainerName = "dtest-prom";
+        public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
+        private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
+        private const string portName = "prom-1";
+        private readonly string config;
+
+        public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
+        {
+            ServicePort = servicePort;
+            PrometheusNumber = prometheusNumber;
+            this.config = config;
+        }
+
+        public int ServicePort { get; }
+        public int PrometheusNumber { get; }
+
+        public string GetDeploymentName()
+        {
+            return "test-prom" + PrometheusNumber;
+        }
+
+        public V1Deployment CreatePrometheusDeployment()
+        {
+            var deploymentSpec = new V1Deployment
+            {
+                ApiVersion = "apps/v1",
+                Metadata =  new V1ObjectMeta
+                {
+                    Name = GetDeploymentName(),
+                    NamespaceProperty = K8sCluster.K8sNamespace
+                },
+                Spec = new V1DeploymentSpec
+                {
+                    Replicas = 1,
+                    Selector = new V1LabelSelector
+                    {
+                        MatchLabels = CreateSelector()
+                    },
+                    Template = new V1PodTemplateSpec
+                    {
+                        Metadata = new V1ObjectMeta
+                        {
+                            Labels = CreateSelector()
+                        },
+                        Spec = new V1PodSpec
+                        {
+                            Containers = new List<V1Container>
+                            {
+                                new V1Container
+                                {
+                                    Name = ContainerName,
+                                    Image = dockerImage,
+                                    Ports = new List<V1ContainerPort>
+                                    {
+                                        new V1ContainerPort
+                                        {
+                                            ContainerPort = 9090,
+                                            Name = portName
+                                        }
+                                    },
+                                    Env = new List<V1EnvVar>
+                                    {
+                                        new V1EnvVar
+                                        {
+                                            Name = "PROM_CONFIG",
+                                            Value = config
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+
+            return deploymentSpec;
+        }
+
+        public V1Service CreatePrometheusService()
+        {
+            var serviceSpec = new V1Service
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = "codex-prom-service" + PrometheusNumber,
+                    NamespaceProperty = K8sCluster.K8sNamespace
+                },
+                Spec = new V1ServiceSpec
+                {
+                    Type = "NodePort",
+                    Selector = CreateSelector(),
+                    Ports = new List<V1ServicePort>
+                    {
+                        new V1ServicePort
+                        {
+                            Name = "prom-service" + PrometheusNumber,
+                            Protocol = "TCP",
+                            Port = 9090,
+                            TargetPort = portName,
+                            NodePort = ServicePort
+                        }
+                    }
+                }
+            };
+
+            return serviceSpec;
+        }
+
+        private Dictionary<string, string> CreateSelector()
+        {
+            return new Dictionary<string, string> { { "test-prom", "dtest-prom" } };
+        }
+    }
+}

CodexDistTestCore/MetricsAccess.cs

@@ -0,0 +1,63 @@
+using NUnit.Framework;
+using NUnit.Framework.Constraints;
+
+namespace CodexDistTestCore
+{
+    public interface IMetricsAccess
+    {
+        void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
+    }
+
+    public class MetricsUnavailable : IMetricsAccess
+    {
+        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
+        {
+            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
+            throw new InvalidOperationException();
+        }
+    }
+
+    public class MetricsAccess : IMetricsAccess
+    {
+        private readonly MetricsQuery query;
+        private readonly OnlineCodexNode node;
+
+        public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
+        {
+            this.query = query;
+            this.node = node;
+        }
+
+        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
+        {
+            var metricSet = GetMetricWithTimeout(metricName, node);
+            var metricValue = metricSet.Values[0].Value;
+            Assert.That(metricValue, constraint, message);
+        }
+
+        private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
+        {
+            var start = DateTime.UtcNow;
+
+            while (true)
+            {
+                var mostRecent = GetMostRecent(metricName, node);
+                if (mostRecent != null) return mostRecent;
+                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
+                {
+                    Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
+                    throw new TimeoutException();
+                }
+
+                Utils.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
+        {
+            var result = query.GetMostRecent(metricName, node);
+            if (result == null) return null;
+            return result.Sets.LastOrDefault();
+        }
+    }
+}

CodexDistTestCore/MetricsAggregator.cs

@@ -0,0 +1,78 @@
+using NUnit.Framework;
+using System.Text;
+
+namespace CodexDistTestCore
+{
+    public class MetricsAggregator
+    {
+        private readonly NumberSource prometheusNumberSource = new NumberSource(0);
+        private readonly TestLog log;
+        private readonly K8sManager k8sManager;
+        private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
+
+        public MetricsAggregator(TestLog log, K8sManager k8sManager)
+        {
+            this.log = log;
+            this.k8sManager = k8sManager;
+        }
+
+        public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
+        {
+            log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
+
+            var config = GeneratePrometheusConfig(nodes);
+            var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
+            var query = new MetricsQuery(prometheus);
+            activePrometheuses.Add(query, nodes);
+
+            log.Log("Metrics service started.");
+
+            foreach(var node in nodes)
+            {
+                node.Metrics = new MetricsAccess(query, node);
+            }
+        }
+
+        public void DownloadAllMetrics()
+        {
+            var download = new MetricsDownloader(log, activePrometheuses);
+            download.DownloadAllMetrics();
+        }
+
+        private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
+        {
+            var config = "";
+            config += "global:\n";
+            config += "  scrape_interval: 30s\n";
+            config += "  scrape_timeout: 10s\n";
+            config += "\n";
+            config += "scrape_configs:\n";
+            config += "  - job_name: services\n";
+            config += "    metrics_path: /metrics\n";
+            config += "    static_configs:\n";
+            config += "      - targets:\n";
+
+            foreach (var node in nodes)
+            {
+                var ip = node.Group.PodInfo!.Ip;
+                var port = node.Container.MetricsPort;
+                config += $"          - '{ip}:{port}'\n";
+            }
+
+            var bytes = Encoding.ASCII.GetBytes(config);
+            return Convert.ToBase64String(bytes);
+        }
+    }
+
+    public class PrometheusInfo
+    {
+        public PrometheusInfo(int servicePort, PodInfo podInfo)
+        {
+            ServicePort = servicePort;
+            PodInfo = podInfo;
+        }
+
+        public int ServicePort { get; }
+        public PodInfo PodInfo { get; }
+    }
+}

CodexDistTestCore/MetricsDownloader.cs

@@ -0,0 +1,97 @@
+using System.Globalization;
+
+namespace CodexDistTestCore
+{
+    public class MetricsDownloader
+    {
+        private readonly TestLog log;
+        private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
+
+        public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
+        {
+            this.log = log;
+            this.activePrometheuses = activePrometheuses;
+        }
+
+        public void DownloadAllMetrics()
+        {
+            foreach (var pair in activePrometheuses)
+            {
+                DownloadAllMetrics(pair.Key, pair.Value);
+            }
+        }
+
+        private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                DownloadAllMetricsForNode(query, node);
+            }
+        }
+
+        private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
+        {
+            var metrics = query.GetAllMetricsForNode(node);
+            if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
+
+            var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
+            var map = CreateValueMap(metrics);
+
+            WriteToFile(node.GetName(), headers, map);
+        }
+
+        private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
+        {
+            var file = log.CreateSubfile("csv");
+            log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
+
+            file.WriteRaw(string.Join(",", headers));
+
+            foreach (var pair in map)
+            {
+                file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
+            }
+        }
+
+        private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
+        {
+            var map = CreateForAllTimestamps(metrics);
+            foreach (var metric in metrics.Sets)
+            {
+                AddToMap(map, metric);
+            }
+            return map;
+
+        }
+
+        private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
+        {
+            var result = new Dictionary<DateTime, List<string>>();
+            var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
+            foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
+            return result;
+        }
+
+        private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
+        {
+            foreach (var key in map.Keys)
+            {
+                map[key].Add(GetValueAtTimestamp(key, metric));
+            }
+        }
+
+        private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
+        {
+            var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
+            if (value == null) return "";
+            return value.Value.ToString(CultureInfo.InvariantCulture);
+        }
+
+        private string FormatTimestamp(DateTime key)
+        {
+            var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+            var diff = key - origin;
+            return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
+        }
+    }
+}

CodexDistTestCore/MetricsQuery.cs

@@ -0,0 +1,190 @@
+using CodexDistTestCore.Config;
+using System.Globalization;
+
+namespace CodexDistTestCore
+{
+    public class MetricsQuery
+    {
+        private readonly K8sCluster k8sCluster = new K8sCluster();
+        private readonly Http http;
+
+        public MetricsQuery(PrometheusInfo prometheusInfo)
+        {
+            http = new Http(
+                k8sCluster.GetIp(),
+                prometheusInfo.ServicePort,
+                "api/v1");
+        }
+
+        public Metrics? GetMostRecent(string metricName, OnlineCodexNode node)
+        {
+            var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
+            if (response == null) return null;
+
+            return new Metrics
+            {
+                Sets = response.data.result.Select(r =>
+                {
+                    return new MetricsSet
+                    {
+                        Instance = r.metric.instance,
+                        Values = MapSingleValue(r.value)
+                    };
+                }).ToArray()
+            };
+        }
+
+        public Metrics? GetMetrics(string metricName)
+        {
+            var response = GetAll(metricName);
+            if (response == null) return null;
+            return MapResponseToMetrics(response);
+        }
+
+        public Metrics? GetAllMetricsForNode(OnlineCodexNode node)
+        {
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
+            if (response.status != "success") return null;
+            return MapResponseToMetrics(response);
+        }
+
+        private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
+        {
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
+            if (response.status != "success") return null;
+            return response;
+        }
+
+        private PrometheusQueryResponse? GetAll(string metricName)
+        {
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
+            if (response.status != "success") return null;
+            return response;
+        }
+
+        private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
+        {
+            return new Metrics
+            {
+                Sets = response.data.result.Select(r =>
+                {
+                    return new MetricsSet
+                    {
+                        Name = r.metric.__name__,
+                        Instance = r.metric.instance,
+                        Values = MapMultipleValues(r.values)
+                    };
+                }).ToArray()
+            };
+        }
+
+        private MetricsSetValue[] MapSingleValue(object[] value)
+        {
+            if (value != null && value.Length > 0)
+            {
+                return new[]
+                {
+                    MapValue(value)
+                };
+            }
+            return Array.Empty<MetricsSetValue>();
+        }
+
+        private MetricsSetValue[] MapMultipleValues(object[][] values)
+        {
+            if (values != null && values.Length > 0)
+            {
+                return values.Select(v => MapValue(v)).ToArray();
+            }
+            return Array.Empty<MetricsSetValue>();
+        }
+
+        private MetricsSetValue MapValue(object[] value)
+        {
+            if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
+
+            return new MetricsSetValue
+            {
+                Timestamp = ToTimestamp(value[0]),
+                Value = ToValue(value[1])
+            };            
+        }
+
+        private string GetInstanceNameForNode(OnlineCodexNode node)
+        {
+            var pod = node.Group.PodInfo!;
+            return $"{pod.Ip}:{node.Container.MetricsPort}";
+        }
+
+        private string GetInstanceStringForNode(OnlineCodexNode node)
+        {
+            return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
+        }
+
+        private string GetQueryTimeRange()
+        {
+            return "[12h]";
+        }
+
+        private double ToValue(object v)
+        {
+            return Convert.ToDouble(v, CultureInfo.InvariantCulture);
+        }
+
+        private DateTime ToTimestamp(object v)
+        {
+            var unixSeconds = ToValue(v);
+            return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
+        }
+    }
+
+    public class Metrics
+    {
+        public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
+    }
+
+    public class MetricsSet
+    {
+        public string Name { get; set; } = string.Empty;
+        public string Instance { get; set; } = string.Empty;
+        public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
+    }
+
+    public class MetricsSetValue
+    {
+        public DateTime Timestamp { get; set; }
+        public double Value { get; set; }
+    }
+
+    public class PrometheusQueryResponse
+    {
+        public string status { get; set; } = string.Empty;
+        public PrometheusQueryResponseData data { get; set; } = new();
+    }
+
+    public class PrometheusQueryResponseData
+    {
+        public string resultType { get; set; } = string.Empty;
+        public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
+    }
+
+    public class PrometheusQueryResponseDataResultEntry
+    {
+        public ResultEntryMetric metric { get; set; } = new();
+        public object[] value { get; set; } = Array.Empty<object>();
+        public object[][] values { get; set; } = Array.Empty<object[]>();
+    }
+
+    public class ResultEntryMetric
+    {
+        public string __name__ { get; set; } = string.Empty;
+        public string instance { get; set; } = string.Empty;
+        public string job { get; set; } = string.Empty;
+    }
+
+    public class PrometheusAllNamesResponse
+    {
+        public string status { get; set; } = string.Empty;
+        public string[] data { get; set; } = Array.Empty<string>();
+    }
+}

CodexDistTestCore/OfflineCodexNodes.cs

@@ -6,6 +6,7 @@
         IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
         IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
         IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
+        IOfflineCodexNodes EnableMetrics();
         ICodexNodeGroup BringOnline();
     }
 
@@ -34,12 +35,14 @@
         public CodexLogLevel? LogLevel { get; private set; }
         public IOnlineCodexNode? BootstrapNode { get; private set; }
         public ByteSize? StorageQuota { get; private set; }
+        public bool MetricsEnabled { get; private set; }
 
         public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
         {
             this.k8SManager = k8SManager;
             NumberOfNodes = numberOfNodes;
             Location = Location.Unspecified;
+            MetricsEnabled = false;
         }
 
         public ICodexNodeGroup BringOnline()
@@ -71,6 +74,12 @@
             return this;
         }
 
+        public IOfflineCodexNodes EnableMetrics()
+        {
+            MetricsEnabled = true;
+            return this;
+        }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());

CodexDistTestCore/OnlineCodexNode.cs

@@ -9,7 +9,8 @@ namespace CodexDistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         void ConnectToPeer(IOnlineCodexNode node);
-        CodexNodeLog DownloadLog();
+        ICodexNodeLog DownloadLog();
+        IMetricsAccess Metrics { get; }
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
@@ -30,6 +31,7 @@ namespace CodexDistTestCore
 
         public CodexNodeContainer Container { get; }
         public CodexNodeGroup Group { get; internal set; } = null!;
+        public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable();
 
         public string GetName()
         {
@@ -80,7 +82,7 @@ namespace CodexDistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public CodexNodeLog DownloadLog()
+        public ICodexNodeLog DownloadLog()
         {
             return Group.DownloadLog(this);
         }

CodexDistTestCore/PodLogDownloader.cs

@@ -8,9 +8,9 @@ namespace CodexDistTestCore
     }
 
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
+    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
     {
-        public DontDownloadLogsOnFailureAttribute()
+        public DontDownloadLogsAndMetricsOnFailureAttribute()
             : base(Timing.UseLongTimeoutsKey)
         {
         }

CodexDistTestCore/TestLog.cs

@@ -39,11 +39,21 @@ namespace CodexDistTestCore
                 Log(result.Message);
                 Log($"{result.StackTrace}");
             }
+
+            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
+            {
+                RenameLogFile();
+            }
         }
 
-        public LogFile CreateSubfile()
+        private void RenameLogFile()
         {
-            return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}");
+            file.ConcatToFilename("_FAILED");
+        }
+
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
         }
 
         private static string GetTestName()
@@ -63,10 +73,17 @@ namespace CodexDistTestCore
 
     public class LogFile
     {
+        private readonly DateTime now;
+        private string name;
+        private readonly string ext;
         private readonly string filepath;
 
-        public LogFile(DateTime now, string name)
+        public LogFile(DateTime now, string name, string ext = "log")
         {
+            this.now = now;
+            this.name = name;
+            this.ext = ext;
+
             filepath = Path.Join(
                 LogConfig.LogRoot,
                 $"{now.Year}-{Pad(now.Month)}",
@@ -74,12 +91,11 @@ namespace CodexDistTestCore
 
             Directory.CreateDirectory(filepath);
 
-            FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
+            GenerateFilename();
-            FullFilename = Path.Combine(filepath, FilenameWithoutPath);
         }
 
-        public string FullFilename { get; }
+        public string FullFilename { get; private set; } = string.Empty;
-        public string FilenameWithoutPath { get; }
+        public string FilenameWithoutPath { get; private set; } = string.Empty;
 
         public void Write(string message)
         {
@@ -98,6 +114,17 @@ namespace CodexDistTestCore
             }
         }
 
+        public void ConcatToFilename(string toAdd)
+        {
+            var oldFullName = FullFilename;
+
+            name += toAdd;
+
+            GenerateFilename();
+
+            File.Move(oldFullName, FullFilename);
+        }
+
         private static string Pad(int n)
         {
             return n.ToString().PadLeft(2, '0');
@@ -107,5 +134,11 @@ namespace CodexDistTestCore
         {
             return $"[{DateTime.UtcNow.ToString("u")}]";
         }
+
+        private void GenerateFilename()
+        {
+            FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}";
+            FullFilename = Path.Combine(filepath, FilenameWithoutPath);
+        }
     }
 }

CodexDistTestCore/Timing.cs

@@ -40,6 +40,11 @@ namespace CodexDistTestCore
             return GetTimes().K8sOperationTimeout();
         }
 
+        public static TimeSpan WaitForMetricTimeout()
+        {
+            return GetTimes().WaitForMetricTimeout();
+        }
+
         private static ITimeSet GetTimes()
         {
             var testProperties = TestContext.CurrentContext.Test.Properties;
@@ -55,6 +60,7 @@ namespace CodexDistTestCore
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
+        TimeSpan WaitForMetricTimeout();
     }
 
     public class DefaultTimeSet : ITimeSet
@@ -83,6 +89,11 @@ namespace CodexDistTestCore
         {
             return TimeSpan.FromMinutes(5);
         }
+
+        public TimeSpan WaitForMetricTimeout()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
     }
 
     public class LongTimeSet : ITimeSet
@@ -111,5 +122,10 @@ namespace CodexDistTestCore
         {
             return TimeSpan.FromMinutes(15);
         }
+
+        public TimeSpan WaitForMetricTimeout()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
     }
 }

LongTests/BasicTests/TestInfraTests.cs

@@ -8,7 +8,9 @@ namespace LongTests.BasicTests
         [Test, UseLongTimeouts]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = SetupCodexNodes(1000).BringOnline();
+            var group = SetupCodexNodes(1000)
+                            .EnableMetrics() // Increases use of port address space per node.
+                            .BringOnline();
 
             var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();
 

Tests/BasicTests/SimpleTests.cs

@@ -20,7 +20,7 @@ namespace Tests.BasicTests
             Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
         }
 
-        [Test, DontDownloadLogsOnFailure]
+        [Test, DontDownloadLogsAndMetricsOnFailure]
         public void CanAccessLogs()
         {
             var node = SetupCodexNodes(1).BringOnline()[0];
@@ -30,6 +30,31 @@ namespace Tests.BasicTests
             log.AssertLogContains("Started codex node");
         }
 
+        [Test]
+        public void TwoMetricsExample()
+        {
+            var group = SetupCodexNodes(2)
+                        .EnableMetrics()
+                        .BringOnline();
+
+            var group2 = SetupCodexNodes(2)
+                        .EnableMetrics()
+                        .BringOnline();
+
+            var primary = group[0];
+            var secondary = group[1];
+            var primary2 = group2[0];
+            var secondary2 = group2[1];
+
+            primary.ConnectToPeer(secondary);
+            primary2.ConnectToPeer(secondary2);
+
+            Thread.Sleep(TimeSpan.FromMinutes(5));
+
+            primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
+            primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
+        }
+
         [Test]
         public void OneClientTest()
         {
@@ -66,6 +91,7 @@ namespace Tests.BasicTests
         }
 
         [Test]
+        [Ignore("Requires Location map to be configured for k8s cluster.")]
         public void TwoClientsTwoLocationsTest()
         {
             var primary = SetupCodexNodes(1)