From 3d1ee8ebdb9bd7a9bd21a3b45dd92620b4c1d34e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 27 Mar 2023 16:24:04 +0200 Subject: [PATCH] Using customized prometheus image to control configuration. This works. --- CodexDistTestCore/CodexDistTestCore.csproj | 1 - CodexDistTestCore/DistTest.cs | 11 ++- CodexDistTestCore/K8sCp.cs | 79 ---------------------- CodexDistTestCore/K8sManager.cs | 13 ++-- CodexDistTestCore/K8sOperations.cs | 17 ++--- CodexDistTestCore/K8sPrometheusSpecs.cs | 25 +++++-- CodexDistTestCore/MetricsAccess.cs | 55 ++++++++++++++- CodexDistTestCore/MetricsAggregator.cs | 76 +++++++++------------ Tests/BasicTests/SimpleTests.cs | 2 - 9 files changed, 118 insertions(+), 161 deletions(-) delete mode 100644 CodexDistTestCore/K8sCp.cs diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index 2b63192..a42e9be 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -12,7 +12,6 @@ - diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 45f1be7..33af574 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -87,11 +87,13 @@ namespace CodexDistTestCore public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes) { - Assert.That(nodes.All(n => HasMetricsEnable(n)), + var onlineNodes = nodes.Cast().ToArray(); + + Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled), "Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " + "To use metrics, please use 'EnableMetrics()' when setting up Codex nodes."); - return metricsAggregator.BeginCollectingMetricsFor(nodes); + return metricsAggregator.BeginCollectingMetricsFor(onlineNodes); } public void AssertWithTimeout(Func operation, T isEqualTo, string message) @@ -150,11 +152,6 @@ namespace CodexDistTestCore var testProperties = TestContext.CurrentContext.Test.Properties; return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); } - - private bool HasMetricsEnable(IOnlineCodexNode n) - { - return ((OnlineCodexNode)n).Group.Origin.MetricsEnabled; - } } public static class GlobalTestFailure diff --git a/CodexDistTestCore/K8sCp.cs b/CodexDistTestCore/K8sCp.cs deleted file mode 100644 index 8faf9ac..0000000 --- a/CodexDistTestCore/K8sCp.cs +++ /dev/null @@ -1,79 +0,0 @@ -using ICSharpCode.SharpZipLib.Tar; -using k8s; -using System.Text; - -namespace CodexDistTestCore -{ - // From: https://github.com/kubernetes-client/csharp/blob/master/examples/cp/Cp.cs - public class K8sCp - { - private readonly Kubernetes client; - - public K8sCp(Kubernetes client) - { - this.client = client; - } - - public async Task CopyFileToPodAsync(string podName, string @namespace, string containerName, Stream inputFileStream, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken)) - { - var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) => - { - var fileInfo = new FileInfo(destinationFilePath); - try - { - using (var memoryStream = new MemoryStream()) - { - using (var tarOutputStream = new TarOutputStream(memoryStream, Encoding.Default)) - { - tarOutputStream.IsStreamOwner = false; - - var fileSize = inputFileStream.Length; - var entry = TarEntry.CreateTarEntry(fileInfo.Name); - - entry.Size = fileSize; - - tarOutputStream.PutNextEntry(entry); - await inputFileStream.CopyToAsync(tarOutputStream); - tarOutputStream.CloseEntry(); - } - - memoryStream.Position = 0; - - await memoryStream.CopyToAsync(stdIn); - await stdIn.FlushAsync(); - } - - } - catch (Exception ex) - { - throw new IOException($"Copy command failed: {ex.Message}"); - } - - using StreamReader streamReader = new StreamReader(stdError); - while (streamReader.EndOfStream == false) - { - string error = await streamReader.ReadToEndAsync(); - throw new IOException($"Copy command failed: {error}"); - } - }); - - string destinationFolder = GetFolderName(destinationFilePath); - - return await client.NamespacedPodExecAsync( - podName, - @namespace, - containerName, - new string[] { "sh", "-c", $"tar xmf - -C {destinationFolder}" }, - false, - handler, - cancellationToken); - } - - private static string GetFolderName(string filePath) - { - var folderName = Path.GetDirectoryName(filePath); - - return string.IsNullOrEmpty(folderName) ? "." : folderName; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 9d39cad..f86cdbf 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -58,16 +58,13 @@ K8s(k => k.FetchPodLog(node, logHandler)); } - public PrometheusInfo BringOnlinePrometheus() + public PrometheusInfo BringOnlinePrometheus(string config) { - PrometheusInfo? info = null; - K8s(k => info = k.BringOnlinePrometheus(codexGroupNumberSource.GetNextServicePort())); - return info!; - } + var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config); - public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) - { - K8s(k => k.UploadFileToPod(podName, containerName, fileStream, destinationPath)); + PrometheusInfo? info = null; + K8s(k => info = k.BringOnlinePrometheus(spec)); + return info!; } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index be9ba3f..7826c7d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -58,22 +58,15 @@ namespace CodexDistTestCore logHandler.Log(stream); } - public PrometheusInfo BringOnlinePrometheus(int servicePort) + public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) { EnsureTestNamespace(); - var spec = new K8sPrometheusSpecs(); CreatePrometheusDeployment(spec); - CreatePrometheusService(spec, servicePort); + CreatePrometheusService(spec); WaitUntilPrometheusOnline(spec); - return new PrometheusInfo(servicePort, FetchNewPod()); - } - - public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) - { - var cp = new K8sCp(client); - Utils.Wait(cp.CopyFileToPodAsync(podName, K8sCluster.K8sNamespace, containerName, fileStream, destinationPath)); + return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } private void FetchPodInfo(CodexNodeGroup online) @@ -201,9 +194,9 @@ namespace CodexDistTestCore online.Service = null; } - private void CreatePrometheusService(K8sPrometheusSpecs spec, int servicePort) + private void CreatePrometheusService(K8sPrometheusSpecs spec) { - client.CreateNamespacedService(spec.CreatePrometheusService(servicePort), K8sNamespace); + client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); } #endregion diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs index c97d60e..1dc8c28 100644 --- a/CodexDistTestCore/K8sPrometheusSpecs.cs +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -7,8 +7,17 @@ namespace CodexDistTestCore { public const string ContainerName = "dtest-prom"; public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; - private const string dockerImage = "prom/prometheus:v2.30.3"; + private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; private const string portName = "prom-1"; + private readonly string config; + + public K8sPrometheusSpecs(int servicePort, string config) + { + ServicePort = servicePort; + this.config = config; + } + + public int ServicePort { get; } public string GetDeploymentName() { @@ -54,10 +63,14 @@ namespace CodexDistTestCore Name = portName } }, - Command = new List + Env = new List { - $"--web.enable-lifecycle --config.file={ConfigFilepath}" - }, + new V1EnvVar + { + Name = "PROM_CONFIG", + Value = config + } + } } } } @@ -68,7 +81,7 @@ namespace CodexDistTestCore return deploymentSpec; } - public V1Service CreatePrometheusService(int servicePort) + public V1Service CreatePrometheusService() { var serviceSpec = new V1Service { @@ -90,7 +103,7 @@ namespace CodexDistTestCore Protocol = "TCP", Port = 9090, TargetPort = portName, - NodePort = servicePort + NodePort = ServicePort } } } diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 33deaca..de8ee01 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Config; + +namespace CodexDistTestCore { public interface IMetricsAccess { @@ -7,9 +9,60 @@ public class MetricsAccess : IMetricsAccess { + private readonly K8sCluster k8sCluster = new K8sCluster(); + private readonly Http http; + + public MetricsAccess(PrometheusInfo prometheusInfo) + { + http = new Http( + k8sCluster.GetIp(), + prometheusInfo.ServicePort, + "api/v1"); + } + public int GetMostRecentInt(string metricName, IOnlineCodexNode node) { + var now = DateTime.UtcNow; + var off = new DateTimeOffset(now); + var nowUnix = off.ToUnixTimeSeconds(); + + var hour = now.AddHours(-1); + var off2 = new DateTimeOffset(hour); + var hourUnix = off2.ToUnixTimeSeconds(); + + var response = http.HttpGetJson($"query_range?query=libp2p_peers&start={hourUnix}&end={nowUnix}&step=100"); + return 0; } } + + public class PrometheusQueryRangeResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryRangeResponseData data { get; set; } = new(); + } + + public class PrometheusQueryRangeResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryRangeResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryRangeResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public ResultEntryValue[] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class ResultEntryValue + { + + } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 273ea0e..594d311 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System.Text; namespace CodexDistTestCore { @@ -6,7 +7,6 @@ namespace CodexDistTestCore { private readonly TestLog log; private readonly K8sManager k8sManager; - private readonly List activeMetricsNodes = new List(); private PrometheusInfo? activePrometheus; public MetricsAggregator(TestLog log, K8sManager k8sManager) @@ -15,71 +15,57 @@ namespace CodexDistTestCore this.k8sManager = k8sManager; } - public MetricsAccess BeginCollectingMetricsFor(IOnlineCodexNode[] nodes) + public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes) { - EnsurePrometheusPod(); + if (activePrometheus != null) + { + Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :("); + throw new InvalidOperationException(); + } - AddNewCodexNodes(nodes); + log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); - // Get IPS and ports from all nodes, format prometheus configuration - var config = GeneratePrometheusConfig(); - // Create config file inside prometheus pod - k8sManager.UploadFileToPod( - activePrometheus!.PodInfo.Name, - K8sPrometheusSpecs.ContainerName, - config, - K8sPrometheusSpecs.ConfigFilepath); + var config = GeneratePrometheusConfig(nodes); + StartPrometheusPod(config); - // HTTP POST request to the /-/reload endpoint (when the --web.enable-lifecycle flag is enabled). - - return new MetricsAccess(); + log.Log("Metrics service started."); + return new MetricsAccess(activePrometheus!); } public void DownloadAllMetrics() { } - private void EnsurePrometheusPod() + private void StartPrometheusPod(string config) { if (activePrometheus != null) return; - activePrometheus = k8sManager.BringOnlinePrometheus(); + activePrometheus = k8sManager.BringOnlinePrometheus(config); } - private void AddNewCodexNodes(IOnlineCodexNode[] nodes) + private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) { - activeMetricsNodes.AddRange(nodes.Where(n => !activeMetricsNodes.Contains(n)).Cast()); - } + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + config += " - 'prometheus:9090'\n"; - private Stream GeneratePrometheusConfig() - { - var stream = new MemoryStream(); - using var writer = new StreamWriter(stream); - - writer.WriteLine("global:"); - writer.WriteLine(" scrape_interval: 30s"); - writer.WriteLine(" scrape_timeout: 10s"); - writer.WriteLine(""); - writer.WriteLine("rule_files:"); - writer.WriteLine(" - alert.yml"); - writer.WriteLine(""); - writer.WriteLine("scrape_configs:"); - writer.WriteLine(" - job_name: services"); - writer.WriteLine(" metrics_path: /metrics"); - writer.WriteLine(" static_configs:"); - writer.WriteLine(" - targets:"); - writer.WriteLine(" - 'prometheus:9090'"); - - foreach (var node in activeMetricsNodes) + foreach (var node in nodes) { var ip = node.Group.PodInfo!.Ip; - var port = node.Container.ServicePort; - writer.WriteLine($" - '{ip}:{port}'"); + var port = node.Container.MetricsPort; + config += $" - '{ip}:{port}'\n"; } - return stream; + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); } - - } public class PrometheusInfo diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 0241add..ba7f15a 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -43,8 +43,6 @@ namespace Tests.BasicTests var secondary = group[1]; primary.ConnectToPeer(secondary); - Thread.Sleep(10000); - AssertWithTimeout( () => metrics.GetMostRecentInt("libp2p_peers", primary), isEqualTo: 1,