From 1ca3ddc67e95623e02fc69de5f7e4b169d4c4154 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 13 Sep 2023 11:25:08 +0200 Subject: [PATCH] Implements metrics plugin --- Core/TimeSet.cs | 6 - DistTestCore/LongTimeSet.cs | 5 - MetricsPlugin/CoreInterfaceExtensions.cs | 21 +- MetricsPlugin/GrafanaContainerRecipe.cs | 25 -- MetricsPlugin/GrafanaStarter.cs | 188 ------------ MetricsPlugin/MetricsAccess.cs | 128 ++++---- MetricsPlugin/MetricsAccessFactory.cs | 35 --- MetricsPlugin/MetricsDownloader.cs | 134 +++++---- MetricsPlugin/MetricsMode.cs | 9 - MetricsPlugin/MetricsPlugin.cs | 17 +- MetricsPlugin/MetricsQuery.cs | 333 ++++++++++----------- MetricsPlugin/MetricsScrapeTarget.cs | 32 ++ MetricsPlugin/PrometheusContainerRecipe.cs | 31 +- MetricsPlugin/PrometheusStarter.cs | 70 +++-- MetricsPlugin/PrometheusStartupConfig.cs | 22 +- Tests/BasicTests/ExampleTests.cs | 3 + Tests/Tests.csproj | 1 + 17 files changed, 424 insertions(+), 636 deletions(-) delete mode 100644 MetricsPlugin/GrafanaContainerRecipe.cs delete mode 100644 MetricsPlugin/GrafanaStarter.cs delete mode 100644 MetricsPlugin/MetricsAccessFactory.cs delete mode 100644 MetricsPlugin/MetricsMode.cs create mode 100644 MetricsPlugin/MetricsScrapeTarget.cs diff --git a/Core/TimeSet.cs b/Core/TimeSet.cs index db78fa5..ff7a19e 100644 --- a/Core/TimeSet.cs +++ b/Core/TimeSet.cs @@ -7,7 +7,6 @@ TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); - TimeSpan WaitForMetricTimeout(); } public class DefaultTimeSet : ITimeSet @@ -36,10 +35,5 @@ { return TimeSpan.FromMinutes(30); } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromSeconds(30); - } } } diff --git a/DistTestCore/LongTimeSet.cs b/DistTestCore/LongTimeSet.cs index 8d0d5e8..cc441cc 100644 --- a/DistTestCore/LongTimeSet.cs +++ b/DistTestCore/LongTimeSet.cs @@ -28,10 +28,5 @@ namespace DistTestCore { return TimeSpan.FromMinutes(15); } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromMinutes(5); - } } } diff --git a/MetricsPlugin/CoreInterfaceExtensions.cs b/MetricsPlugin/CoreInterfaceExtensions.cs index 1f6cecd..0346857 100644 --- a/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/MetricsPlugin/CoreInterfaceExtensions.cs @@ -1,13 +1,30 @@ using Core; using KubernetesWorkflow; +using Logging; namespace MetricsPlugin { public static class CoreInterfaceExtensions { - public static RunningContainer StartMetricsCollector(this CoreInterface ci, RunningContainers[] scrapeTargets) + public static RunningContainers StartMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) { - return null!;// Plugin(ci).StartMetricsCollector(scrapeTargets); + return Plugin(ci).StartMetricsCollector(scrapeTargets); + } + + public static IMetricsAccess GetMetricsFor(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget) + { + return Plugin(ci).CreateAccessForTarget(metricsContainer, scrapeTarget); + } + + public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) + { + var rc = ci.StartMetricsCollector(scrapeTargets); + return scrapeTargets.Select(t => ci.GetMetricsFor(rc, t)).ToArray(); + } + + public static LogFile? DownloadAllMetrics(this CoreInterface ci, IMetricsAccess metricsAccess, string targetName) + { + return Plugin(ci).DownloadAllMetrics(metricsAccess, targetName); } private static MetricsPlugin Plugin(CoreInterface ci) diff --git a/MetricsPlugin/GrafanaContainerRecipe.cs b/MetricsPlugin/GrafanaContainerRecipe.cs deleted file mode 100644 index db38461..0000000 --- a/MetricsPlugin/GrafanaContainerRecipe.cs +++ /dev/null @@ -1,25 +0,0 @@ -//using KubernetesWorkflow; - -//namespace DistTestCore.Metrics -//{ -// public class GrafanaContainerRecipe : DefaultContainerRecipe -// { -// public override string AppName => "grafana"; -// public override string Image => "grafana/grafana-oss:10.0.3"; - -// public const string DefaultAdminUser = "adminium"; -// public const string DefaultAdminPassword = "passwordium"; - -// protected override void InitializeRecipe(StartupConfig startupConfig) -// { -// AddExposedPort(3000); - -// AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); -// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org."); -// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor"); - -// AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser); -// AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword); -// } -// } -//} diff --git a/MetricsPlugin/GrafanaStarter.cs b/MetricsPlugin/GrafanaStarter.cs deleted file mode 100644 index 7a30a65..0000000 --- a/MetricsPlugin/GrafanaStarter.cs +++ /dev/null @@ -1,188 +0,0 @@ -using KubernetesWorkflow; - -namespace MetricsPlugin -{ - public class GrafanaStarter - { - private const string StorageQuotaThresholdReplaceToken = "\"\""; - private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"\""; - - public GrafanaStarter() - { - } - - public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)//, CodexSetup codexSetup) - { - return null!; - //LogStart($"Starting dashboard server"); - - //var grafanaContainer = StartGrafanaContainer(); - //var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer); - - //var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth); - - //Log("Connecting datasource..."); - //AddDataSource(http, prometheusContainer); - - //Log("Uploading dashboard configurations..."); - //var jsons = ReadEachDashboardJsonFile(codexSetup); - //var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray(); - - //LogEnd("Dashboard server started."); - - //return new GrafanaStartInfo(dashboardUrls, grafanaContainer); - } - - //private RunningContainer StartGrafanaContainer() - //{ - // var startupConfig = new StartupConfig(); - - // var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); - // var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); - // if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created."); - - // return grafanaContainers.Containers.First(); - //} - - //private void AddBasicAuth(HttpClient client) - //{ - // client.SetBasicAuthentication( - // GrafanaContainerRecipe.DefaultAdminUser, - // GrafanaContainerRecipe.DefaultAdminPassword); - //} - - //private static void AddDataSource(Http http, RunningContainer prometheusContainer) - //{ - // var prometheusAddress = prometheusContainer.ClusterExternalAddress; - // var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port; - // var response = http.HttpPostJson("datasources", new GrafanaDataSourceRequest - // { - // uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", - // name = "CodexPrometheus", - // type = "prometheus", - // url = prometheusUrl, - // access = "proxy", - // basicAuth = false, - // jsonData = new GrafanaDataSourceJsonData - // { - // httpMethod = "POST" - // } - // }); - - // if (response.message != "Datasource added") - // { - // throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message); - // } - //} - - //public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson) - //{ - // var request = GetDashboardCreateRequest(dashboardJson); - // var response = http.HttpPostString("dashboards/db", request); - // var jsonResponse = JsonConvert.DeserializeObject(response); - // if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard."); - - // var grafanaAddress = grafanaContainer.ClusterExternalAddress; - // return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; - //} - - //private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup) - //{ - // var assembly = Assembly.GetExecutingAssembly(); - // var resourceNames = new[] - // { - // "DistTestCore.Metrics.dashboard.json" - // }; - - // return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray(); - //} - - //private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup) - //{ - // using var stream = assembly.GetManifestResourceStream(resourceName); - // if (stream == null) throw new Exception("Unable to find resource " + resourceName); - // using var reader = new StreamReader(stream); - // return ApplyReplacements(reader.ReadToEnd(), codexSetup); - //} - - //private static string ApplyReplacements(string input, CodexSetup codexSetup) - //{ - // var quotaString = GetQuotaString(codexSetup); - // var softMaxString = GetSoftMaxString(codexSetup); - - // return input - // .Replace(StorageQuotaThresholdReplaceToken, quotaString) - // .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString); - //} - - //private static string GetQuotaString(CodexSetup codexSetup) - //{ - // return GetCodexStorageQuotaInBytes(codexSetup).ToString(); - //} - - //private static string GetSoftMaxString(CodexSetup codexSetup) - //{ - // var quota = GetCodexStorageQuotaInBytes(codexSetup); - // var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing. - // return softMax.ToString(); - //} - - //private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup) - //{ - // if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes; - - // // Codex default: 8GB - // return 8.GB().SizeInBytes; - //} - - //private static string GetDashboardCreateRequest(string dashboardJson) - //{ - // return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; - //} - } - - public class GrafanaStartInfo - { - public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container) - { - DashboardUrls = dashboardUrls; - Container = container; - } - - public string[] DashboardUrls { get; } - public RunningContainer Container { get; } - } - - public class GrafanaDataSourceRequest - { - public string uid { get; set; } = string.Empty; - public string name { get; set; } = string.Empty; - public string type { get; set; } = string.Empty; - public string url { get; set; } = string.Empty; - public string access { get; set; } = string.Empty; - public bool basicAuth { get; set; } - public GrafanaDataSourceJsonData jsonData { get; set; } = new(); - } - - public class GrafanaDataSourceResponse - { - public int id { get; set; } - public string message { get; set; } = string.Empty; - public string name { get; set; } = string.Empty; - } - - public class GrafanaDataSourceJsonData - { - public string httpMethod { get; set; } = string.Empty; - } - - public class GrafanaPostDashboardResponse - { - public int id { get; set; } - public string slug { get; set; } = string.Empty; - public string status { get; set; } = string.Empty; - public string uid { get; set; } = string.Empty; - public string url { get; set; } = string.Empty; - public int version { get; set; } - } -} diff --git a/MetricsPlugin/MetricsAccess.cs b/MetricsPlugin/MetricsAccess.cs index 3641d4f..beb5ed1 100644 --- a/MetricsPlugin/MetricsAccess.cs +++ b/MetricsPlugin/MetricsAccess.cs @@ -1,81 +1,69 @@ -//using DistTestCore.Helpers; -//using KubernetesWorkflow; -//using Logging; -//using NUnit.Framework; -//using NUnit.Framework.Constraints; -//using Utils; +using Utils; -//namespace DistTestCore.Metrics -//{ -// public interface IMetricsAccess -// { -// void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); -// } +namespace MetricsPlugin +{ + public interface IMetricsAccess + { + Metrics? GetAllMetrics(); + MetricsSet GetMetric(string metricName); + MetricsSet GetMetric(string metricName, TimeSpan timeout); + } -// public class MetricsAccess : IMetricsAccess -// { -// private readonly BaseLog log; -// private readonly ITimeSet timeSet; -// private readonly MetricsQuery query; -// private readonly RunningContainer node; + public class MetricsAccess : IMetricsAccess + { + private readonly MetricsQuery query; + private readonly IMetricsScrapeTarget target; -// public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) -// { -// this.log = log; -// this.timeSet = timeSet; -// this.query = query; -// this.node = node; -// } + public MetricsAccess(MetricsQuery query, IMetricsScrapeTarget target) + { + this.query = query; + this.target = target; + } -// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") -// { -// AssertHelpers.RetryAssert(constraint, () => -// { -// var metricSet = GetMetricWithTimeout(metricName); -// var metricValue = metricSet.Values[0].Value; + //public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + //{ + // AssertHelpers.RetryAssert(constraint, () => + // { + // var metricSet = GetMetricWithTimeout(metricName); + // var metricValue = metricSet.Values[0].Value; -// log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); -// return metricValue; -// }, message); -// } + // log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); + // return metricValue; + // }, message); + //} -// public Metrics? GetAllMetrics() -// { -// return query.GetAllMetricsForNode(node); -// } + public Metrics? GetAllMetrics() + { + return query.GetAllMetricsForNode(target); + } -// private MetricsSet GetMetricWithTimeout(string metricName) -// { -// var start = DateTime.UtcNow; + public MetricsSet GetMetric(string metricName) + { + return GetMetric(metricName, TimeSpan.FromSeconds(10)); + } -// while (true) -// { -// var mostRecent = GetMostRecent(metricName); -// if (mostRecent != null) return mostRecent; -// if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout()) -// { -// Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); -// throw new TimeoutException(); -// } + public MetricsSet GetMetric(string metricName, TimeSpan timeout) + { + var start = DateTime.UtcNow; -// Time.Sleep(TimeSpan.FromSeconds(2)); -// } -// } + while (true) + { + var mostRecent = GetMostRecent(metricName); + if (mostRecent != null) return mostRecent; + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException(); + } -// private MetricsSet? GetMostRecent(string metricName) -// { -// var result = query.GetMostRecent(metricName, node); -// if (result == null) return null; -// return result.Sets.LastOrDefault(); -// } -// } + Time.Sleep(TimeSpan.FromSeconds(2)); + } + } -// public class MetricsUnavailable : IMetricsAccess -// { -// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") -// { -// Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); -// throw new InvalidOperationException(); -// } -// } -//} + private MetricsSet? GetMostRecent(string metricName) + { + var result = query.GetMostRecent(metricName, target); + if (result == null) return null; + return result.Sets.LastOrDefault(); + } + } +} diff --git a/MetricsPlugin/MetricsAccessFactory.cs b/MetricsPlugin/MetricsAccessFactory.cs deleted file mode 100644 index c185fef..0000000 --- a/MetricsPlugin/MetricsAccessFactory.cs +++ /dev/null @@ -1,35 +0,0 @@ -//using KubernetesWorkflow; - -//namespace DistTestCore.Metrics -//{ -// public interface IMetricsAccessFactory -// { -// IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer); -// } - -// public class MetricsUnavailableAccessFactory : IMetricsAccessFactory -// { -// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) -// { -// return new MetricsUnavailable(); -// } -// } - -// public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory -// { -// private readonly TestLifecycle lifecycle; -// private readonly RunningContainers prometheusContainer; - -// public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer) -// { -// this.lifecycle = lifecycle; -// this.prometheusContainer = prometheusContainer; -// } - -// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) -// { -// var query = new MetricsQuery(lifecycle, prometheusContainer); -// return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer); -// } -// } -//} diff --git a/MetricsPlugin/MetricsDownloader.cs b/MetricsPlugin/MetricsDownloader.cs index 5efa085..1c642a0 100644 --- a/MetricsPlugin/MetricsDownloader.cs +++ b/MetricsPlugin/MetricsDownloader.cs @@ -1,80 +1,82 @@ -//using Logging; -//using System.Globalization; +using Logging; +using System.Globalization; -//namespace DistTestCore.Metrics -//{ -// public class MetricsDownloader -// { -// private readonly BaseLog log; +namespace MetricsPlugin +{ + public class MetricsDownloader + { + private readonly ILog log; -// public MetricsDownloader(BaseLog log) -// { -// this.log = log; -// } + public MetricsDownloader(ILog log) + { + this.log = log; + } -// public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access) -// { -// var metrics = access.GetAllMetrics(); -// if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + public LogFile? DownloadAllMetrics(string targetName, IMetricsAccess access) + { + var metrics = access.GetAllMetrics(); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return null; -// var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); -// var map = CreateValueMap(metrics); + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); -// WriteToFile(nodeName, headers, map); -// } + return WriteToFile(targetName, headers, map); + } -// private void WriteToFile(string nodeName, string[] headers, Dictionary> map) -// { -// var file = log.CreateSubfile("csv"); -// log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}"); + private LogFile WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}"); -// file.WriteRaw(string.Join(",", headers)); + file.WriteRaw(string.Join(",", headers)); -// foreach (var pair in map) -// { -// file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); -// } -// } + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } -// private Dictionary> CreateValueMap(Metrics metrics) -// { -// var map = CreateForAllTimestamps(metrics); -// foreach (var metric in metrics.Sets) -// { -// AddToMap(map, metric); -// } -// return map; + return file; + } -// } + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; -// private Dictionary> CreateForAllTimestamps(Metrics metrics) -// { -// var result = new Dictionary>(); -// var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); -// foreach (var timestamp in timestamps) result.Add(timestamp, new List()); -// return result; -// } + } -// private void AddToMap(Dictionary> map, MetricsSet metric) -// { -// foreach (var key in map.Keys) -// { -// map[key].Add(GetValueAtTimestamp(key, metric)); -// } -// } + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } -// private string GetValueAtTimestamp(DateTime key, MetricsSet metric) -// { -// var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); -// if (value == null) return ""; -// return value.Value.ToString(CultureInfo.InvariantCulture); -// } + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } -// private string FormatTimestamp(DateTime key) -// { -// var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); -// var diff = key - origin; -// return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); -// } -// } -//} + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/MetricsPlugin/MetricsMode.cs b/MetricsPlugin/MetricsMode.cs deleted file mode 100644 index 44a99e9..0000000 --- a/MetricsPlugin/MetricsMode.cs +++ /dev/null @@ -1,9 +0,0 @@ -//namespace DistTestCore.Metrics -//{ -// public enum MetricsMode -// { -// None, -// Record, -// Dashboard -// } -//} diff --git a/MetricsPlugin/MetricsPlugin.cs b/MetricsPlugin/MetricsPlugin.cs index fc79541..c2ed11c 100644 --- a/MetricsPlugin/MetricsPlugin.cs +++ b/MetricsPlugin/MetricsPlugin.cs @@ -1,5 +1,6 @@ using Core; using KubernetesWorkflow; +using Logging; namespace MetricsPlugin { @@ -14,19 +15,29 @@ namespace MetricsPlugin starter = new PrometheusStarter(tools); } - public void Announce() { - //log.Log("Hi from the metrics plugin."); + tools.GetLog().Log("Hi from the metrics plugin."); } public void Decommission() { } - public RunningContainers StartMetricsCollector(RunningContainers[] scrapeTargets) + public RunningContainers StartMetricsCollector(IMetricsScrapeTarget[] scrapeTargets) { return starter.CollectMetricsFor(scrapeTargets); } + + public MetricsAccess CreateAccessForTarget(RunningContainers runningContainers, IMetricsScrapeTarget target) + { + return starter.CreateAccessForTarget(runningContainers, target); + } + + public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName) + { + var downloader = new MetricsDownloader(tools.GetLog()); + return downloader.DownloadAllMetrics(targetName, metricsAccess); + } } } diff --git a/MetricsPlugin/MetricsQuery.cs b/MetricsPlugin/MetricsQuery.cs index 668ee9f..8d75f20 100644 --- a/MetricsPlugin/MetricsQuery.cs +++ b/MetricsPlugin/MetricsQuery.cs @@ -1,198 +1,189 @@ -//using DistTestCore.Codex; -//using KubernetesWorkflow; -//using System.Globalization; +using Core; +using KubernetesWorkflow; +using System.Globalization; -//namespace DistTestCore.Metrics -//{ -// public class MetricsQuery -// { -// private readonly Http http; +namespace MetricsPlugin +{ + public class MetricsQuery + { + private readonly Http http; -// public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers) -// { -// RunningContainers = runningContainers; + public MetricsQuery(IPluginTools tools, RunningContainers runningContainers) + { + RunningContainers = runningContainers; + http = tools.CreateHttp(runningContainers.Containers[0].Address, "api/v1"); + } -// var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]); + public RunningContainers RunningContainers { get; } -// http = new Http( -// lifecycle.Log, -// lifecycle.TimeSet, -// address, -// "api/v1"); -// } + public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target) + { + var response = GetLastOverTime(metricName, GetInstanceStringForNode(target)); + if (response == null) return null; -// public RunningContainers RunningContainers { get; } + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } -// public Metrics? GetMostRecent(string metricName, RunningContainer node) -// { -// var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); -// if (response == null) return null; + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + return MapResponseToMetrics(response); + } -// return new Metrics -// { -// Sets = response.data.result.Select(r => -// { -// return new MetricsSet -// { -// Instance = r.metric.instance, -// Values = MapSingleValue(r.value) -// }; -// }).ToArray() -// }; -// } + public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target) + { + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } -// public Metrics? GetMetrics(string metricName) -// { -// var response = GetAll(metricName); -// if (response == null) return null; -// return MapResponseToMetrics(response); -// } + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } -// public Metrics? GetAllMetricsForNode(RunningContainer node) -// { -// var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); -// if (response.status != "success") return null; -// return MapResponseToMetrics(response); -// } + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } -// private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) -// { -// var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); -// if (response.status != "success") return null; -// return response; -// } + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } -// private PrometheusQueryResponse? GetAll(string metricName) -// { -// var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); -// if (response.status != "success") return null; -// return response; -// } + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } -// private Metrics MapResponseToMetrics(PrometheusQueryResponse response) -// { -// return new Metrics -// { -// Sets = response.data.result.Select(r => -// { -// return new MetricsSet -// { -// Name = r.metric.__name__, -// Instance = r.metric.instance, -// Values = MapMultipleValues(r.values) -// }; -// }).ToArray() -// }; -// } + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } -// private MetricsSetValue[] MapSingleValue(object[] value) -// { -// if (value != null && value.Length > 0) -// { -// return new[] -// { -// MapValue(value) -// }; -// } -// return Array.Empty(); -// } + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); -// private MetricsSetValue[] MapMultipleValues(object[][] values) -// { -// if (values != null && values.Length > 0) -// { -// return values.Select(v => MapValue(v)).ToArray(); -// } -// return Array.Empty(); -// } + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } -// private MetricsSetValue MapValue(object[] value) -// { -// if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + private string GetInstanceNameForNode(IMetricsScrapeTarget target) + { + return $"{target.Ip}:{target.Port}"; + } -// return new MetricsSetValue -// { -// Timestamp = ToTimestamp(value[0]), -// Value = ToValue(value[1]) -// }; -// } + private string GetInstanceStringForNode(IMetricsScrapeTarget target) + { + return "{instance=\"" + GetInstanceNameForNode(target) + "\"}"; + } -// private string GetInstanceNameForNode(RunningContainer node) -// { -// var ip = node.Pod.PodInfo.Ip; -// var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; -// return $"{ip}:{port}"; -// } + private string GetQueryTimeRange() + { + return "[12h]"; + } -// private string GetInstanceStringForNode(RunningContainer node) -// { -// return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; -// } + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } -// private string GetQueryTimeRange() -// { -// return "[12h]"; -// } + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } -// private double ToValue(object v) -// { -// return Convert.ToDouble(v, CultureInfo.InvariantCulture); -// } + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } -// private DateTime ToTimestamp(object v) -// { -// var unixSeconds = ToValue(v); -// return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); -// } -// } + public class MetricsSet + { + public string Name { get; set; } = string.Empty; + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } -// public class Metrics -// { -// public MetricsSet[] Sets { get; set; } = Array.Empty(); -// } + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } -// public class MetricsSet -// { -// public string Name { get; set; } = string.Empty; -// public string Instance { get; set; } = string.Empty; -// public MetricsSetValue[] Values { get; set; } = Array.Empty(); -// } + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } -// public class MetricsSetValue -// { -// public DateTime Timestamp { get; set; } -// public double Value { get; set; } -// } + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } -// public class PrometheusQueryResponse -// { -// public string status { get; set; } = string.Empty; -// public PrometheusQueryResponseData data { get; set; } = new(); -// } + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } -// public class PrometheusQueryResponseData -// { -// public string resultType { get; set; } = string.Empty; -// public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); -// } + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } -// public class PrometheusQueryResponseDataResultEntry -// { -// public ResultEntryMetric metric { get; set; } = new(); -// public object[] value { get; set; } = Array.Empty(); -// public object[][] values { get; set; } = Array.Empty(); -// } - -// public class ResultEntryMetric -// { -// public string __name__ { get; set; } = string.Empty; -// public string instance { get; set; } = string.Empty; -// public string job { get; set; } = string.Empty; -// } - -// public class PrometheusAllNamesResponse -// { -// public string status { get; set; } = string.Empty; -// public string[] data { get; set; } = Array.Empty(); -// } -//} + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } +} diff --git a/MetricsPlugin/MetricsScrapeTarget.cs b/MetricsPlugin/MetricsScrapeTarget.cs new file mode 100644 index 0000000..6dff1be --- /dev/null +++ b/MetricsPlugin/MetricsScrapeTarget.cs @@ -0,0 +1,32 @@ +using KubernetesWorkflow; + +namespace MetricsPlugin +{ + public interface IMetricsScrapeTarget + { + string Ip { get; } + int Port { get; } + } + + public class MetricsScrapeTarget : IMetricsScrapeTarget + { + public MetricsScrapeTarget(string ip, int port) + { + Ip = ip; + Port = port; + } + + public MetricsScrapeTarget(RunningContainer container, int port) + : this(container.Pod.PodInfo.Ip, port) + { + } + + public MetricsScrapeTarget(RunningContainer container, Port port) + : this(container, port.Number) + { + } + + public string Ip { get; } + public int Port { get; } + } +} diff --git a/MetricsPlugin/PrometheusContainerRecipe.cs b/MetricsPlugin/PrometheusContainerRecipe.cs index c3b5706..584ad82 100644 --- a/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/MetricsPlugin/PrometheusContainerRecipe.cs @@ -1,18 +1,19 @@ -//using KubernetesWorkflow; +using Core; +using KubernetesWorkflow; -//namespace DistTestCore.Metrics -//{ -// public class PrometheusContainerRecipe : DefaultContainerRecipe -// { -// public override string AppName => "prometheus"; -// public override string Image => "codexstorage/dist-tests-prometheus:latest"; +namespace MetricsPlugin +{ + public class PrometheusContainerRecipe : DefaultContainerRecipe + { + public override string AppName => "prometheus"; + public override string Image => "codexstorage/dist-tests-prometheus:latest"; -// protected override void InitializeRecipe(StartupConfig startupConfig) -// { -// var config = startupConfig.Get(); + protected override void InitializeRecipe(StartupConfig startupConfig) + { + var config = startupConfig.Get(); -// AddExposedPortAndVar("PROM_PORT"); -// AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); -// } -// } -//} + AddExposedPortAndVar("PROM_PORT"); + AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + } + } +} diff --git a/MetricsPlugin/PrometheusStarter.cs b/MetricsPlugin/PrometheusStarter.cs index 1edc8fe..37814ea 100644 --- a/MetricsPlugin/PrometheusStarter.cs +++ b/MetricsPlugin/PrometheusStarter.cs @@ -1,5 +1,6 @@ using Core; using KubernetesWorkflow; +using System.Text; namespace MetricsPlugin { @@ -12,42 +13,51 @@ namespace MetricsPlugin this.tools = tools; } - public RunningContainers CollectMetricsFor(RunningContainers[] containers) + public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets) { - //LogStart($"Starting metrics server for {containers.Describe()}"); - //var startupConfig = new StartupConfig(); - //startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); + Log($"Starting metrics server for {targets.Length} targets..."); + var startupConfig = new StartupConfig(); + startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets))); - //var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); - //var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); - //if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + var workflow = tools.CreateWorkflow(); + var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); + if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); - //return runningContainers; - return null!; + Log("Metrics server started."); + return runningContainers; } - //private string GeneratePrometheusConfig(RunningContainer[] nodes) - //{ - // var config = ""; - // config += "global:\n"; - // config += " scrape_interval: 10s\n"; - // config += " scrape_timeout: 10s\n"; - // config += "\n"; - // config += "scrape_configs:\n"; - // config += " - job_name: services\n"; - // config += " metrics_path: /metrics\n"; - // config += " static_configs:\n"; - // config += " - targets:\n"; + public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target) + { + var metricsQuery = new MetricsQuery(tools, metricsContainer); + return new MetricsAccess(metricsQuery, target); + } - // foreach (var node in nodes) - // { - // var ip = node.Pod.PodInfo.Ip; - // var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; - // config += $" - '{ip}:{port}'\n"; - // } + private void Log(string msg) + { + tools.GetLog().Log(msg); + } - // var bytes = Encoding.ASCII.GetBytes(config); - // return Convert.ToBase64String(bytes); - //} + private static string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets) + { + var config = ""; + config += "global:\n"; + config += " scrape_interval: 10s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + + foreach (var target in targets) + { + config += $" - '{target.Ip}:{target.Port}'\n"; + } + + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); + } } } diff --git a/MetricsPlugin/PrometheusStartupConfig.cs b/MetricsPlugin/PrometheusStartupConfig.cs index 57434eb..a490420 100644 --- a/MetricsPlugin/PrometheusStartupConfig.cs +++ b/MetricsPlugin/PrometheusStartupConfig.cs @@ -1,12 +1,12 @@ -//namespace DistTestCore.Metrics -//{ -// public class PrometheusStartupConfig -// { -// public PrometheusStartupConfig(string prometheusConfigBase64) -// { -// PrometheusConfigBase64 = prometheusConfigBase64; -// } +namespace MetricsPlugin +{ + public class PrometheusStartupConfig + { + public PrometheusStartupConfig(string prometheusConfigBase64) + { + PrometheusConfigBase64 = prometheusConfigBase64; + } -// public string PrometheusConfigBase64 { get; } -// } -//} + public string PrometheusConfigBase64 { get; } + } +} diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 15c82f5..0b5156d 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,5 +1,6 @@ using CodexPlugin; using DistTestCore; +using MetricsPlugin; using NUnit.Framework; using Utils; @@ -23,6 +24,8 @@ namespace Tests.BasicTests [Test] public void TwoMetricsExample() { + var rc = Ci.StartMetricsCollector(); + //var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); //var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index ecc4f58..27552d7 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -15,6 +15,7 @@ +