diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index fa0e348..4a1f239 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -2,7 +2,7 @@ { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) { Name = name; ServicePort = servicePort; @@ -12,6 +12,7 @@ DiscoveryPort = discoveryPort; ListenPort = listenPort; DataDir = dataDir; + MetricsPort = metricsPort; } public string Name { get; } @@ -22,6 +23,7 @@ public int DiscoveryPort { get; } public int ListenPort { get; } public string DataDir { get; } + public int MetricsPort { get; } } public class CodexGroupNumberSource @@ -57,7 +59,7 @@ this.groupContainerFactory = groupContainerFactory; } - public CodexNodeContainer CreateNext() + public CodexNodeContainer CreateNext(OfflineCodexNodes offline) { var n = containerNameSource.GetNextNumber(); return new CodexNodeContainer( @@ -68,8 +70,15 @@ containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), listenPort: codexPortSource.GetNextNumber(), - dataDir: $"datadir{n}" + dataDir: $"datadir{n}", + metricsPort: GetMetricsPort(offline) ); } + + private int GetMetricsPort(OfflineCodexNodes offline) + { + if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); + return 0; + } } } diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs index b4cae47..1a0572a 100644 --- a/CodexDistTestCore/CodexNodeLog.cs +++ b/CodexDistTestCore/CodexNodeLog.cs @@ -2,7 +2,12 @@ namespace CodexDistTestCore { - public class CodexNodeLog + public interface ICodexNodeLog + { + void AssertLogContains(string expectedString); + } + + public class CodexNodeLog : ICodexNodeLog { private readonly LogFile logFile; diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs index 7cd938e..333ec0d 100644 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ b/CodexDistTestCore/Config/CodexDockerImage.cs @@ -25,12 +25,12 @@ namespace CodexDistTestCore.Config { public List Result { get; } = new List(); - public void Create(OfflineCodexNodes node, CodexNodeContainer environment) + public void Create(OfflineCodexNodes node, CodexNodeContainer container) { - AddVar("API_PORT", environment.ApiPort.ToString()); - AddVar("DATA_DIR", environment.DataDir); - AddVar("DISC_PORT", environment.DiscoveryPort.ToString()); - AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}"); + AddVar("API_PORT", container.ApiPort.ToString()); + AddVar("DATA_DIR", container.DataDir); + AddVar("DISC_PORT", container.DiscoveryPort.ToString()); + AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}"); if (node.BootstrapNode != null) { @@ -45,6 +45,11 @@ namespace CodexDistTestCore.Config { AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!); } + if (node.MetricsEnabled) + { + AddVar("METRICS_ADDR", "0.0.0.0"); + AddVar("METRICS_PORT", container.MetricsPort.ToString()); + } } private void AddVar(string key, string value) diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index 0fd60d0..a290193 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -17,7 +17,8 @@ namespace CodexDistTestCore.Config public KubernetesClientConfiguration GetK8sClientConfig() { if (config != null) return config; - config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); return config; } diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 1331c6a..db32409 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -57,7 +57,7 @@ namespace CodexDistTestCore try { log.EndTest(); - IncludeLogsOnTestFailure(); + IncludeLogsAndMetricsOnTestFailure(); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } @@ -78,19 +78,20 @@ namespace CodexDistTestCore return new OfflineCodexNodes(k8sManager, numberOfNodes); } - private void IncludeLogsOnTestFailure() + private void IncludeLogsAndMetricsOnTestFailure() { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - if (IsDownloadingLogsEnabled()) + if (IsDownloadingLogsAndMetricsEnabled()) { - log.Log("Downloading all CodexNode logs because of test failure..."); + log.Log("Downloading all CodexNode logs and metrics because of test failure..."); k8sManager.ForEachOnlineGroup(DownloadLogs); + k8sManager.DownloadAllMetrics(); } else { - log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute."); + log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); } } } @@ -105,7 +106,7 @@ namespace CodexDistTestCore } } - private bool IsDownloadingLogsEnabled() + private bool IsDownloadingLogsAndMetricsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index 6a2daf7..6fbd55f 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -80,28 +80,30 @@ namespace CodexDistTestCore return info.Length; } - public void AssertIsEqual(TestFile? other) + public void AssertIsEqual(TestFile? actual) { - if (other == null) Assert.Fail("TestFile is null."); - if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - var bytes1 = new byte[FileManager.ChunkSize]; - var bytes2 = new byte[FileManager.ChunkSize]; + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - var read1 = 0; - var read2 = 0; + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; while (true) { - read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize); - read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize); + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - if (read1 == 0 && read2 == 0) return; - Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length."); - CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal."); + if (readExpected == 0 && readActual == 0) return; + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); } } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 22ec144..f92c628 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -14,11 +14,13 @@ private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly TestLog log; private readonly IFileManager fileManager; + private readonly MetricsAggregator metricsAggregator; public K8sManager(TestLog log, IFileManager fileManager) { this.log = log; this.fileManager = fileManager; + metricsAggregator = new MetricsAggregator(log, this); } public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) @@ -29,6 +31,11 @@ log.Log($"{online.Describe()} online."); + if (offline.MetricsEnabled) + { + BringOnlineMetrics(online); + } + return online; } @@ -58,20 +65,41 @@ K8s(k => k.FetchPodLog(node, logHandler)); } + public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) + { + var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); + + PrometheusInfo? info = null; + K8s(k => info = k.BringOnlinePrometheus(spec)); + return info!; + } + + public void DownloadAllMetrics() + { + metricsAggregator.DownloadAllMetrics(); + } + + private void BringOnlineMetrics(CodexNodeGroup group) + { + var onlineNodes = group.Nodes.Cast().ToArray(); + + metricsAggregator.BeginCollectingMetricsFor(onlineNodes); + } + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { - var containers = CreateContainers(offline.NumberOfNodes); + var containers = CreateContainers(offline); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); onlineCodexNodeGroups.Add(result); return result; } - private CodexNodeContainer[] CreateContainers(int number) + private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline) { var factory = new CodexNodeContainerFactory(codexGroupNumberSource); var containers = new List(); - for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); + for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline)); return containers.ToArray(); } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index a9f1613..7826c7d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -1,5 +1,6 @@ using CodexDistTestCore.Config; using k8s; +using k8s.KubeConfigModels; using k8s.Models; using NUnit.Framework; @@ -57,7 +58,23 @@ namespace CodexDistTestCore logHandler.Log(stream); } + public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) + { + EnsureTestNamespace(); + + CreatePrometheusDeployment(spec); + CreatePrometheusService(spec); + WaitUntilPrometheusOnline(spec); + + return new PrometheusInfo(spec.ServicePort, FetchNewPod()); + } + private void FetchPodInfo(CodexNodeGroup online) + { + online.PodInfo = FetchNewPod(); + } + + private PodInfo FetchNewPod() { var pods = client.ListNamespacedPod(K8sNamespace).Items; @@ -65,12 +82,13 @@ namespace CodexDistTestCore Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); var newPod = newPods.Single(); - online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP); + var info = new PodInfo(newPod.Name(), newPod.Status.PodIP); - Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure."); - Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure."); knownPods.Add(newPod.Name()); + return info; } #region Waiting @@ -103,6 +121,16 @@ namespace CodexDistTestCore WaitUntil(() => !IsTestNamespaceOnline()); } + private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) + { + var deploymentName = spec.GetDeploymentName(); + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; + }); + } + private void WaitUntil(Func predicate) { var start = DateTime.UtcNow; @@ -166,6 +194,11 @@ namespace CodexDistTestCore online.Service = null; } + private void CreatePrometheusService(K8sPrometheusSpecs spec) + { + client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); + } + #endregion #region Deployment management @@ -232,6 +265,7 @@ namespace CodexDistTestCore Env = dockerImage.CreateEnvironmentVariables(offline, container) }); } + return result; } @@ -242,6 +276,11 @@ namespace CodexDistTestCore online.Deployment = null; } + private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) + { + client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); + } + #endregion #region Namespace management diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs new file mode 100644 index 0000000..dcda941 --- /dev/null +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -0,0 +1,122 @@ +using CodexDistTestCore.Config; +using k8s.Models; + +namespace CodexDistTestCore +{ + public class K8sPrometheusSpecs + { + public const string ContainerName = "dtest-prom"; + public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; + private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; + private const string portName = "prom-1"; + private readonly string config; + + public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) + { + ServicePort = servicePort; + PrometheusNumber = prometheusNumber; + this.config = config; + } + + public int ServicePort { get; } + public int PrometheusNumber { get; } + + public string GetDeploymentName() + { + return "test-prom" + PrometheusNumber; + } + + public V1Deployment CreatePrometheusDeployment() + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetDeploymentName(), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateSelector() + }, + Spec = new V1PodSpec + { + Containers = new List + { + new V1Container + { + Name = ContainerName, + Image = dockerImage, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = 9090, + Name = portName + } + }, + Env = new List + { + new V1EnvVar + { + Name = "PROM_CONFIG", + Value = config + } + } + } + } + } + } + } + }; + + return deploymentSpec; + } + + public V1Service CreatePrometheusService() + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = "codex-prom-service" + PrometheusNumber, + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = CreateSelector(), + Ports = new List + { + new V1ServicePort + { + Name = "prom-service" + PrometheusNumber, + Protocol = "TCP", + Port = 9090, + TargetPort = portName, + NodePort = ServicePort + } + } + } + }; + + return serviceSpec; + } + + private Dictionary CreateSelector() + { + return new Dictionary { { "test-prom", "dtest-prom" } }; + } + } +} diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs new file mode 100644 index 0000000..feb3fe7 --- /dev/null +++ b/CodexDistTestCore/MetricsAccess.cs @@ -0,0 +1,63 @@ +using NUnit.Framework; +using NUnit.Framework.Constraints; + +namespace CodexDistTestCore +{ + public interface IMetricsAccess + { + void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); + } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } + + public class MetricsAccess : IMetricsAccess + { + private readonly MetricsQuery query; + private readonly OnlineCodexNode node; + + public MetricsAccess(MetricsQuery query, OnlineCodexNode node) + { + this.query = query; + this.node = node; + } + + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + var metricSet = GetMetricWithTimeout(metricName, node); + var metricValue = metricSet.Values[0].Value; + Assert.That(metricValue, constraint, message); + } + + private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) + { + var start = DateTime.UtcNow; + + while (true) + { + var mostRecent = GetMostRecent(metricName, node); + if (mostRecent != null) return mostRecent; + if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + { + Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); + throw new TimeoutException(); + } + + Utils.Sleep(TimeSpan.FromSeconds(2)); + } + } + + private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) + { + var result = query.GetMostRecent(metricName, node); + if (result == null) return null; + return result.Sets.LastOrDefault(); + } + } +} diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs new file mode 100644 index 0000000..59e884d --- /dev/null +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -0,0 +1,78 @@ +using NUnit.Framework; +using System.Text; + +namespace CodexDistTestCore +{ + public class MetricsAggregator + { + private readonly NumberSource prometheusNumberSource = new NumberSource(0); + private readonly TestLog log; + private readonly K8sManager k8sManager; + private readonly Dictionary activePrometheuses = new Dictionary(); + + public MetricsAggregator(TestLog log, K8sManager k8sManager) + { + this.log = log; + this.k8sManager = k8sManager; + } + + public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes) + { + log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); + + var config = GeneratePrometheusConfig(nodes); + var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); + var query = new MetricsQuery(prometheus); + activePrometheuses.Add(query, nodes); + + log.Log("Metrics service started."); + + foreach(var node in nodes) + { + node.Metrics = new MetricsAccess(query, node); + } + } + + public void DownloadAllMetrics() + { + var download = new MetricsDownloader(log, activePrometheuses); + download.DownloadAllMetrics(); + } + + private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) + { + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + + foreach (var node in nodes) + { + var ip = node.Group.PodInfo!.Ip; + var port = node.Container.MetricsPort; + config += $" - '{ip}:{port}'\n"; + } + + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); + } + } + + public class PrometheusInfo + { + public PrometheusInfo(int servicePort, PodInfo podInfo) + { + ServicePort = servicePort; + PodInfo = podInfo; + } + + public int ServicePort { get; } + public PodInfo PodInfo { get; } + } +} diff --git a/CodexDistTestCore/MetricsDownloader.cs b/CodexDistTestCore/MetricsDownloader.cs new file mode 100644 index 0000000..22ef526 --- /dev/null +++ b/CodexDistTestCore/MetricsDownloader.cs @@ -0,0 +1,97 @@ +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/MetricsQuery.cs new file mode 100644 index 0000000..d8ba684 --- /dev/null +++ b/CodexDistTestCore/MetricsQuery.cs @@ -0,0 +1,190 @@ +using CodexDistTestCore.Config; +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsQuery + { + private readonly K8sCluster k8sCluster = new K8sCluster(); + private readonly Http http; + + public MetricsQuery(PrometheusInfo prometheusInfo) + { + http = new Http( + k8sCluster.GetIp(), + prometheusInfo.ServicePort, + "api/v1"); + } + + public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) + { + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + return MapResponseToMetrics(response); + } + + public Metrics? GetAllMetricsForNode(OnlineCodexNode node) + { + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetInstanceNameForNode(OnlineCodexNode node) + { + var pod = node.Group.PodInfo!; + return $"{pod.Ip}:{node.Container.MetricsPort}"; + } + + private string GetInstanceStringForNode(OnlineCodexNode node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Name { get; set; } = string.Empty; + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } +} diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index 9092618..f5527af 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -6,6 +6,7 @@ IOfflineCodexNodes WithLogLevel(CodexLogLevel level); IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); + IOfflineCodexNodes EnableMetrics(); ICodexNodeGroup BringOnline(); } @@ -34,12 +35,14 @@ public CodexLogLevel? LogLevel { get; private set; } public IOnlineCodexNode? BootstrapNode { get; private set; } public ByteSize? StorageQuota { get; private set; } + public bool MetricsEnabled { get; private set; } public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) { this.k8SManager = k8SManager; NumberOfNodes = numberOfNodes; Location = Location.Unspecified; + MetricsEnabled = false; } public ICodexNodeGroup BringOnline() @@ -71,6 +74,12 @@ return this; } + public IOfflineCodexNodes EnableMetrics() + { + MetricsEnabled = true; + return this; + } + public string Describe() { var args = string.Join(',', DescribeArgs()); diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index f8cf90f..a9cf511 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -9,7 +9,8 @@ namespace CodexDistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); - CodexNodeLog DownloadLog(); + ICodexNodeLog DownloadLog(); + IMetricsAccess Metrics { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -30,6 +31,7 @@ namespace CodexDistTestCore public CodexNodeContainer Container { get; } public CodexNodeGroup Group { get; internal set; } = null!; + public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); public string GetName() { @@ -80,7 +82,7 @@ namespace CodexDistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - public CodexNodeLog DownloadLog() + public ICodexNodeLog DownloadLog() { return Group.DownloadLog(this); } diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs index ce379d7..4df84d2 100644 --- a/CodexDistTestCore/PodLogDownloader.cs +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -8,9 +8,9 @@ namespace CodexDistTestCore } [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsOnFailureAttribute : PropertyAttribute + public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute { - public DontDownloadLogsOnFailureAttribute() + public DontDownloadLogsAndMetricsOnFailureAttribute() : base(Timing.UseLongTimeoutsKey) { } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index aca6f1b..3947678 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -39,11 +39,21 @@ namespace CodexDistTestCore Log(result.Message); Log($"{result.StackTrace}"); } + + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + RenameLogFile(); + } } - public LogFile CreateSubfile() + private void RenameLogFile() { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}"); + file.ConcatToFilename("_FAILED"); + } + + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -63,10 +73,17 @@ namespace CodexDistTestCore public class LogFile { + private readonly DateTime now; + private string name; + private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name) + public LogFile(DateTime now, string name, string ext = "log") { + this.now = now; + this.name = name; + this.ext = ext; + filepath = Path.Join( LogConfig.LogRoot, $"{now.Year}-{Pad(now.Month)}", @@ -74,12 +91,11 @@ namespace CodexDistTestCore Directory.CreateDirectory(filepath); - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); + GenerateFilename(); } - public string FullFilename { get; } - public string FilenameWithoutPath { get; } + public string FullFilename { get; private set; } = string.Empty; + public string FilenameWithoutPath { get; private set; } = string.Empty; public void Write(string message) { @@ -98,6 +114,17 @@ namespace CodexDistTestCore } } + public void ConcatToFilename(string toAdd) + { + var oldFullName = FullFilename; + + name += toAdd; + + GenerateFilename(); + + File.Move(oldFullName, FullFilename); + } + private static string Pad(int n) { return n.ToString().PadLeft(2, '0'); @@ -107,5 +134,11 @@ namespace CodexDistTestCore { return $"[{DateTime.UtcNow.ToString("u")}]"; } + + private void GenerateFilename() + { + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; + FullFilename = Path.Combine(filepath, FilenameWithoutPath); + } } } diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs index 7ee0edf..cfa9456 100644 --- a/CodexDistTestCore/Timing.cs +++ b/CodexDistTestCore/Timing.cs @@ -40,6 +40,11 @@ namespace CodexDistTestCore return GetTimes().K8sOperationTimeout(); } + public static TimeSpan WaitForMetricTimeout() + { + return GetTimes().WaitForMetricTimeout(); + } + private static ITimeSet GetTimes() { var testProperties = TestContext.CurrentContext.Test.Properties; @@ -55,6 +60,7 @@ namespace CodexDistTestCore TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); + TimeSpan WaitForMetricTimeout(); } public class DefaultTimeSet : ITimeSet @@ -83,6 +89,11 @@ namespace CodexDistTestCore { return TimeSpan.FromMinutes(5); } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromSeconds(30); + } } public class LongTimeSet : ITimeSet @@ -111,5 +122,10 @@ namespace CodexDistTestCore { return TimeSpan.FromMinutes(15); } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromMinutes(5); + } } } diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 1e2f62e..705b4fc 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -8,7 +8,9 @@ namespace LongTests.BasicTests [Test, UseLongTimeouts] public void TestInfraShouldHave1000AddressSpacesPerPod() { - var group = SetupCodexNodes(1000).BringOnline(); + var group = SetupCodexNodes(1000) + .EnableMetrics() // Increases use of port address space per node. + .BringOnline(); var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 01e671d..ae72183 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -20,7 +20,7 @@ namespace Tests.BasicTests Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); } - [Test, DontDownloadLogsOnFailure] + [Test, DontDownloadLogsAndMetricsOnFailure] public void CanAccessLogs() { var node = SetupCodexNodes(1).BringOnline()[0]; @@ -30,6 +30,31 @@ namespace Tests.BasicTests log.AssertLogContains("Started codex node"); } + [Test] + public void TwoMetricsExample() + { + var group = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); + + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); + + var primary = group[0]; + var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; + + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); + + Thread.Sleep(TimeSpan.FromMinutes(5)); + + primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + } + [Test] public void OneClientTest() { @@ -66,6 +91,7 @@ namespace Tests.BasicTests } [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] public void TwoClientsTwoLocationsTest() { var primary = SetupCodexNodes(1)