status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implements metrics plugin

This commit is contained in:
benbierens 2023-09-13 11:25:08 +02:00
parent a05a82c030
commit 1ca3ddc67e
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
17 changed files with 424 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Core/TimeSet.cs
View File

@ -7,7 +7,6 @@
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
}
public class DefaultTimeSet : ITimeSet
@ -36,10 +35,5 @@
{
return TimeSpan.FromMinutes(30);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromSeconds(30);
}
}
}

5
DistTestCore/LongTimeSet.cs
View File

@ -28,10 +28,5 @@ namespace DistTestCore
{
return TimeSpan.FromMinutes(15);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromMinutes(5);
}
}
}

21
MetricsPlugin/CoreInterfaceExtensions.cs
View File

@ -1,13 +1,30 @@
using Core;
using KubernetesWorkflow;
using Logging;
namespace MetricsPlugin
{
public static class CoreInterfaceExtensions
{
public static RunningContainer StartMetricsCollector(this CoreInterface ci, RunningContainers[] scrapeTargets)
public static RunningContainers StartMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
{
return null!;// Plugin(ci).StartMetricsCollector(scrapeTargets);
return Plugin(ci).StartMetricsCollector(scrapeTargets);
}
public static IMetricsAccess GetMetricsFor(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
{
return Plugin(ci).CreateAccessForTarget(metricsContainer, scrapeTarget);
}
public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
{
var rc = ci.StartMetricsCollector(scrapeTargets);
return scrapeTargets.Select(t => ci.GetMetricsFor(rc, t)).ToArray();
}
public static LogFile? DownloadAllMetrics(this CoreInterface ci, IMetricsAccess metricsAccess, string targetName)
{
return Plugin(ci).DownloadAllMetrics(metricsAccess, targetName);
}
private static MetricsPlugin Plugin(CoreInterface ci)

25
MetricsPlugin/GrafanaContainerRecipe.cs
View File

@ -1,25 +0,0 @@
//using KubernetesWorkflow;
//namespace DistTestCore.Metrics
//{
// public class GrafanaContainerRecipe : DefaultContainerRecipe
// {
// public override string AppName => "grafana";
// public override string Image => "grafana/grafana-oss:10.0.3";
// public const string DefaultAdminUser = "adminium";
// public const string DefaultAdminPassword = "passwordium";
// protected override void InitializeRecipe(StartupConfig startupConfig)
// {
// AddExposedPort(3000);
// AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
// AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
// AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
// }
// }
//}

188
MetricsPlugin/GrafanaStarter.cs
View File

@ -1,188 +0,0 @@
using KubernetesWorkflow;
namespace MetricsPlugin
{
public class GrafanaStarter
{
private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
public GrafanaStarter()
{
}
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)//, CodexSetup codexSetup)
{
return null!;
//LogStart($"Starting dashboard server");
//var grafanaContainer = StartGrafanaContainer();
//var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
//var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
//Log("Connecting datasource...");
//AddDataSource(http, prometheusContainer);
//Log("Uploading dashboard configurations...");
//var jsons = ReadEachDashboardJsonFile(codexSetup);
//var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
//LogEnd("Dashboard server started.");
//return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
}
//private RunningContainer StartGrafanaContainer()
//{
// var startupConfig = new StartupConfig();
// var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
// var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
// if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
// return grafanaContainers.Containers.First();
//}
//private void AddBasicAuth(HttpClient client)
//{
// client.SetBasicAuthentication(
// GrafanaContainerRecipe.DefaultAdminUser,
// GrafanaContainerRecipe.DefaultAdminPassword);
//}
//private static void AddDataSource(Http http, RunningContainer prometheusContainer)
//{
// var prometheusAddress = prometheusContainer.ClusterExternalAddress;
// var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
// var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
// {
// uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
// name = "CodexPrometheus",
// type = "prometheus",
// url = prometheusUrl,
// access = "proxy",
// basicAuth = false,
// jsonData = new GrafanaDataSourceJsonData
// {
// httpMethod = "POST"
// }
// });
// if (response.message != "Datasource added")
// {
// throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
// }
//}
//public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
//{
// var request = GetDashboardCreateRequest(dashboardJson);
// var response = http.HttpPostString("dashboards/db", request);
// var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
// if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
// var grafanaAddress = grafanaContainer.ClusterExternalAddress;
// return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
//}
//private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
//{
// var assembly = Assembly.GetExecutingAssembly();
// var resourceNames = new[]
// {
// "DistTestCore.Metrics.dashboard.json"
// };
// return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
//}
//private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
//{
// using var stream = assembly.GetManifestResourceStream(resourceName);
// if (stream == null) throw new Exception("Unable to find resource " + resourceName);
// using var reader = new StreamReader(stream);
// return ApplyReplacements(reader.ReadToEnd(), codexSetup);
//}
//private static string ApplyReplacements(string input, CodexSetup codexSetup)
//{
// var quotaString = GetQuotaString(codexSetup);
// var softMaxString = GetSoftMaxString(codexSetup);
// return input
// .Replace(StorageQuotaThresholdReplaceToken, quotaString)
// .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
//}
//private static string GetQuotaString(CodexSetup codexSetup)
//{
// return GetCodexStorageQuotaInBytes(codexSetup).ToString();
//}
//private static string GetSoftMaxString(CodexSetup codexSetup)
//{
// var quota = GetCodexStorageQuotaInBytes(codexSetup);
// var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
// return softMax.ToString();
//}
//private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
//{
// if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
// // Codex default: 8GB
// return 8.GB().SizeInBytes;
//}
//private static string GetDashboardCreateRequest(string dashboardJson)
//{
// return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
//}
}
public class GrafanaStartInfo
{
public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
{
DashboardUrls = dashboardUrls;
Container = container;
}
public string[] DashboardUrls { get; }
public RunningContainer Container { get; }
}
public class GrafanaDataSourceRequest
{
public string uid { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
public string type { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public string access { get; set; } = string.Empty;
public bool basicAuth { get; set; }
public GrafanaDataSourceJsonData jsonData { get; set; } = new();
}
public class GrafanaDataSourceResponse
{
public int id { get; set; }
public string message { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
}
public class GrafanaDataSourceJsonData
{
public string httpMethod { get; set; } = string.Empty;
}
public class GrafanaPostDashboardResponse
{
public int id { get; set; }
public string slug { get; set; } = string.Empty;
public string status { get; set; } = string.Empty;
public string uid { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public int version { get; set; }
}
}

128
MetricsPlugin/MetricsAccess.cs
View File

@ -1,81 +1,69 @@
//using DistTestCore.Helpers;
//using KubernetesWorkflow;
//using Logging;
//using NUnit.Framework;
//using NUnit.Framework.Constraints;
//using Utils;
using Utils;
//namespace DistTestCore.Metrics
//{
// public interface IMetricsAccess
// {
// void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
// }
namespace MetricsPlugin
{
public interface IMetricsAccess
{
Metrics? GetAllMetrics();
MetricsSet GetMetric(string metricName);
MetricsSet GetMetric(string metricName, TimeSpan timeout);
}
// public class MetricsAccess : IMetricsAccess
// {
// private readonly BaseLog log;
// private readonly ITimeSet timeSet;
// private readonly MetricsQuery query;
// private readonly RunningContainer node;
public class MetricsAccess : IMetricsAccess
{
private readonly MetricsQuery query;
private readonly IMetricsScrapeTarget target;
// public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
// {
// this.log = log;
// this.timeSet = timeSet;
// this.query = query;
// this.node = node;
// }
public MetricsAccess(MetricsQuery query, IMetricsScrapeTarget target)
{
this.query = query;
this.target = target;
}
// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
// {
// AssertHelpers.RetryAssert(constraint, () =>
// {
// var metricSet = GetMetricWithTimeout(metricName);
// var metricValue = metricSet.Values[0].Value;
//public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
//{
// AssertHelpers.RetryAssert(constraint, () =>
// {
// var metricSet = GetMetricWithTimeout(metricName);
// var metricValue = metricSet.Values[0].Value;
// log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
// return metricValue;
// }, message);
// }
// log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
// return metricValue;
// }, message);
//}
// public Metrics? GetAllMetrics()
// {
// return query.GetAllMetricsForNode(node);
// }
public Metrics? GetAllMetrics()
{
return query.GetAllMetricsForNode(target);
}
// private MetricsSet GetMetricWithTimeout(string metricName)
// {
// var start = DateTime.UtcNow;
public MetricsSet GetMetric(string metricName)
{
return GetMetric(metricName, TimeSpan.FromSeconds(10));
}
// while (true)
// {
// var mostRecent = GetMostRecent(metricName);
// if (mostRecent != null) return mostRecent;
// if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
// {
// Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
// throw new TimeoutException();
// }
public MetricsSet GetMetric(string metricName, TimeSpan timeout)
{
var start = DateTime.UtcNow;
// Time.Sleep(TimeSpan.FromSeconds(2));
// }
// }
while (true)
{
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException();
}
// private MetricsSet? GetMostRecent(string metricName)
// {
// var result = query.GetMostRecent(metricName, node);
// if (result == null) return null;
// return result.Sets.LastOrDefault();
// }
// }
Time.Sleep(TimeSpan.FromSeconds(2));
}
}
// public class MetricsUnavailable : IMetricsAccess
// {
// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
// {
// Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
// throw new InvalidOperationException();
// }
// }
//}
private MetricsSet? GetMostRecent(string metricName)
{
var result = query.GetMostRecent(metricName, target);
if (result == null) return null;
return result.Sets.LastOrDefault();
}
}
}

35
MetricsPlugin/MetricsAccessFactory.cs
View File

@ -1,35 +0,0 @@
//using KubernetesWorkflow;
//namespace DistTestCore.Metrics
//{
// public interface IMetricsAccessFactory
// {
// IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
// }
// public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
// {
// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
// {
// return new MetricsUnavailable();
// }
// }
// public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
// {
// private readonly TestLifecycle lifecycle;
// private readonly RunningContainers prometheusContainer;
// public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
// {
// this.lifecycle = lifecycle;
// this.prometheusContainer = prometheusContainer;
// }
// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
// {
// var query = new MetricsQuery(lifecycle, prometheusContainer);
// return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
// }
// }
//}

134
MetricsPlugin/MetricsDownloader.cs
View File

@ -1,80 +1,82 @@
//using Logging;
//using System.Globalization;
using Logging;
using System.Globalization;
//namespace DistTestCore.Metrics
//{
// public class MetricsDownloader
// {
// private readonly BaseLog log;
namespace MetricsPlugin
{
public class MetricsDownloader
{
private readonly ILog log;
// public MetricsDownloader(BaseLog log)
// {
// this.log = log;
// }
public MetricsDownloader(ILog log)
{
this.log = log;
}
// public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access)
// {
// var metrics = access.GetAllMetrics();
// if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
public LogFile? DownloadAllMetrics(string targetName, IMetricsAccess access)
{
var metrics = access.GetAllMetrics();
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return null;
// var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
// var map = CreateValueMap(metrics);
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
var map = CreateValueMap(metrics);
// WriteToFile(nodeName, headers, map);
// }
return WriteToFile(targetName, headers, map);
}
// private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
// {
// var file = log.CreateSubfile("csv");
// log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
private LogFile WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
// file.WriteRaw(string.Join(",", headers));
file.WriteRaw(string.Join(",", headers));
// foreach (var pair in map)
// {
// file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
// }
// }
foreach (var pair in map)
{
file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
}
// private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
// {
// var map = CreateForAllTimestamps(metrics);
// foreach (var metric in metrics.Sets)
// {
// AddToMap(map, metric);
// }
// return map;
return file;
}
// }
private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
{
var map = CreateForAllTimestamps(metrics);
foreach (var metric in metrics.Sets)
{
AddToMap(map, metric);
}
return map;
// private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
// {
// var result = new Dictionary<DateTime, List<string>>();
// var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
// foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
// return result;
// }
}
// private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
// {
// foreach (var key in map.Keys)
// {
// map[key].Add(GetValueAtTimestamp(key, metric));
// }
// }
private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
{
var result = new Dictionary<DateTime, List<string>>();
var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
return result;
}
// private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
// {
// var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
// if (value == null) return "";
// return value.Value.ToString(CultureInfo.InvariantCulture);
// }
private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
{
foreach (var key in map.Keys)
{
map[key].Add(GetValueAtTimestamp(key, metric));
}
}
// private string FormatTimestamp(DateTime key)
// {
// var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
// var diff = key - origin;
// return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
// }
// }
//}
private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
{
var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
if (value == null) return "";
return value.Value.ToString(CultureInfo.InvariantCulture);
}
private string FormatTimestamp(DateTime key)
{
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
var diff = key - origin;
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
}
}
}

9
MetricsPlugin/MetricsMode.cs
View File

@ -1,9 +0,0 @@
//namespace DistTestCore.Metrics
//{
// public enum MetricsMode
// {
// None,
// Record,
// Dashboard
// }
//}

17
MetricsPlugin/MetricsPlugin.cs
View File

@ -1,5 +1,6 @@
using Core;
using KubernetesWorkflow;
using Logging;
namespace MetricsPlugin
{
@ -14,19 +15,29 @@ namespace MetricsPlugin
starter = new PrometheusStarter(tools);
}
public void Announce()
{
//log.Log("Hi from the metrics plugin.");
tools.GetLog().Log("Hi from the metrics plugin.");
}
public void Decommission()
{
}
public RunningContainers StartMetricsCollector(RunningContainers[] scrapeTargets)
public RunningContainers StartMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
{
return starter.CollectMetricsFor(scrapeTargets);
}
public MetricsAccess CreateAccessForTarget(RunningContainers runningContainers, IMetricsScrapeTarget target)
{
return starter.CreateAccessForTarget(runningContainers, target);
}
public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)
{
var downloader = new MetricsDownloader(tools.GetLog());
return downloader.DownloadAllMetrics(targetName, metricsAccess);
}
}
}

333
MetricsPlugin/MetricsQuery.cs
View File

@ -1,198 +1,189 @@
//using DistTestCore.Codex;
//using KubernetesWorkflow;
//using System.Globalization;
using Core;
using KubernetesWorkflow;
using System.Globalization;
//namespace DistTestCore.Metrics
//{
// public class MetricsQuery
// {
// private readonly Http http;
namespace MetricsPlugin
{
public class MetricsQuery
{
private readonly Http http;
// public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
// {
// RunningContainers = runningContainers;
public MetricsQuery(IPluginTools tools, RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = tools.CreateHttp(runningContainers.Containers[0].Address, "api/v1");
}
// var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
public RunningContainers RunningContainers { get; }
// http = new Http(
// lifecycle.Log,
// lifecycle.TimeSet,
// address,
// "api/v1");
// }
public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
{
var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
if (response == null) return null;
// public RunningContainers RunningContainers { get; }
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapSingleValue(r.value)
};
}).ToArray()
};
}
// public Metrics? GetMostRecent(string metricName, RunningContainer node)
// {
// var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
// if (response == null) return null;
public Metrics? GetMetrics(string metricName)
{
var response = GetAll(metricName);
if (response == null) return null;
return MapResponseToMetrics(response);
}
// return new Metrics
// {
// Sets = response.data.result.Select(r =>
// {
// return new MetricsSet
// {
// Instance = r.metric.instance,
// Values = MapSingleValue(r.value)
// };
// }).ToArray()
// };
// }
public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return MapResponseToMetrics(response);
}
// public Metrics? GetMetrics(string metricName)
// {
// var response = GetAll(metricName);
// if (response == null) return null;
// return MapResponseToMetrics(response);
// }
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
if (response.status != "success") return null;
return response;
}
// public Metrics? GetAllMetricsForNode(RunningContainer node)
// {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
// if (response.status != "success") return null;
// return MapResponseToMetrics(response);
// }
private PrometheusQueryResponse? GetAll(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return response;
}
// private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
// {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
// if (response.status != "success") return null;
// return response;
// }
private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
{
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Name = r.metric.__name__,
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
// private PrometheusQueryResponse? GetAll(string metricName)
// {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
// if (response.status != "success") return null;
// return response;
// }
private MetricsSetValue[] MapSingleValue(object[] value)
{
if (value != null && value.Length > 0)
{
return new[]
{
MapValue(value)
};
}
return Array.Empty<MetricsSetValue>();
}
// private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
// {
// return new Metrics
// {
// Sets = response.data.result.Select(r =>
// {
// return new MetricsSet
// {
// Name = r.metric.__name__,
// Instance = r.metric.instance,
// Values = MapMultipleValues(r.values)
// };
// }).ToArray()
// };
// }
private MetricsSetValue[] MapMultipleValues(object[][] values)
{
if (values != null && values.Length > 0)
{
return values.Select(v => MapValue(v)).ToArray();
}
return Array.Empty<MetricsSetValue>();
}
// private MetricsSetValue[] MapSingleValue(object[] value)
// {
// if (value != null && value.Length > 0)
// {
// return new[]
// {
// MapValue(value)
// };
// }
// return Array.Empty<MetricsSetValue>();
// }
private MetricsSetValue MapValue(object[] value)
{
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
// private MetricsSetValue[] MapMultipleValues(object[][] values)
// {
// if (values != null && values.Length > 0)
// {
// return values.Select(v => MapValue(v)).ToArray();
// }
// return Array.Empty<MetricsSetValue>();
// }
return new MetricsSetValue
{
Timestamp = ToTimestamp(value[0]),
Value = ToValue(value[1])
};
}
// private MetricsSetValue MapValue(object[] value)
// {
// if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
private string GetInstanceNameForNode(IMetricsScrapeTarget target)
{
return $"{target.Ip}:{target.Port}";
}
// return new MetricsSetValue
// {
// Timestamp = ToTimestamp(value[0]),
// Value = ToValue(value[1])
// };
// }
private string GetInstanceStringForNode(IMetricsScrapeTarget target)
{
return "{instance=\"" + GetInstanceNameForNode(target) + "\"}";
}
// private string GetInstanceNameForNode(RunningContainer node)
// {
// var ip = node.Pod.PodInfo.Ip;
// var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
// return $"{ip}:{port}";
// }
private string GetQueryTimeRange()
{
return "[12h]";
}
// private string GetInstanceStringForNode(RunningContainer node)
// {
// return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
// }
private double ToValue(object v)
{
return Convert.ToDouble(v, CultureInfo.InvariantCulture);
}
// private string GetQueryTimeRange()
// {
// return "[12h]";
// }
private DateTime ToTimestamp(object v)
{
var unixSeconds = ToValue(v);
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
}
// private double ToValue(object v)
// {
// return Convert.ToDouble(v, CultureInfo.InvariantCulture);
// }
public class Metrics
{
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
}
// private DateTime ToTimestamp(object v)
// {
// var unixSeconds = ToValue(v);
// return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
// }
// }
public class MetricsSet
{
public string Name { get; set; } = string.Empty;
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
}
// public class Metrics
// {
// public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
// }
public class MetricsSetValue
{
public DateTime Timestamp { get; set; }
public double Value { get; set; }
}
// public class MetricsSet
// {
// public string Name { get; set; } = string.Empty;
// public string Instance { get; set; } = string.Empty;
// public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
// }
public class PrometheusQueryResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryResponseData data { get; set; } = new();
}
// public class MetricsSetValue
// {
// public DateTime Timestamp { get; set; }
// public double Value { get; set; }
// }
public class PrometheusQueryResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
}
// public class PrometheusQueryResponse
// {
// public string status { get; set; } = string.Empty;
// public PrometheusQueryResponseData data { get; set; } = new();
// }
public class PrometheusQueryResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public object[] value { get; set; } = Array.Empty<object>();
public object[][] values { get; set; } = Array.Empty<object[]>();
}
// public class PrometheusQueryResponseData
// {
// public string resultType { get; set; } = string.Empty;
// public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
// }
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
// public class PrometheusQueryResponseDataResultEntry
// {
// public ResultEntryMetric metric { get; set; } = new();
// public object[] value { get; set; } = Array.Empty<object>();
// public object[][] values { get; set; } = Array.Empty<object[]>();
// }
// public class ResultEntryMetric
// {
// public string __name__ { get; set; } = string.Empty;
// public string instance { get; set; } = string.Empty;
// public string job { get; set; } = string.Empty;
// }
// public class PrometheusAllNamesResponse
// {
// public string status { get; set; } = string.Empty;
// public string[] data { get; set; } = Array.Empty<string>();
// }
//}
public class PrometheusAllNamesResponse
{
public string status { get; set; } = string.Empty;
public string[] data { get; set; } = Array.Empty<string>();
}
}

32
MetricsPlugin/MetricsScrapeTarget.cs Normal file
View File

@ -0,0 +1,32 @@
using KubernetesWorkflow;
namespace MetricsPlugin
{
public interface IMetricsScrapeTarget
{
string Ip { get; }
int Port { get; }
}
public class MetricsScrapeTarget : IMetricsScrapeTarget
{
public MetricsScrapeTarget(string ip, int port)
{
Ip = ip;
Port = port;
}
public MetricsScrapeTarget(RunningContainer container, int port)
: this(container.Pod.PodInfo.Ip, port)
{
}
public MetricsScrapeTarget(RunningContainer container, Port port)
: this(container, port.Number)
{
}
public string Ip { get; }
public int Port { get; }
}
}

31
MetricsPlugin/PrometheusContainerRecipe.cs
View File

@ -1,18 +1,19 @@
//using KubernetesWorkflow;
using Core;
using KubernetesWorkflow;
//namespace DistTestCore.Metrics
//{
// public class PrometheusContainerRecipe : DefaultContainerRecipe
// {
// public override string AppName => "prometheus";
// public override string Image => "codexstorage/dist-tests-prometheus:latest";
namespace MetricsPlugin
{
public class PrometheusContainerRecipe : DefaultContainerRecipe
{
public override string AppName => "prometheus";
public override string Image => "codexstorage/dist-tests-prometheus:latest";
// protected override void InitializeRecipe(StartupConfig startupConfig)
// {
// var config = startupConfig.Get<PrometheusStartupConfig>();
protected override void InitializeRecipe(StartupConfig startupConfig)
{
var config = startupConfig.Get<PrometheusStartupConfig>();
// AddExposedPortAndVar("PROM_PORT");
// AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
// }
// }
//}
AddExposedPortAndVar("PROM_PORT");
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
}
}
}

70
MetricsPlugin/PrometheusStarter.cs
View File

@ -1,5 +1,6 @@
using Core;
using KubernetesWorkflow;
using System.Text;
namespace MetricsPlugin
{
@ -12,42 +13,51 @@ namespace MetricsPlugin
this.tools = tools;
}
public RunningContainers CollectMetricsFor(RunningContainers[] containers)
public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
{
//LogStart($"Starting metrics server for {containers.Describe()}");
//var startupConfig = new StartupConfig();
//startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
Log($"Starting metrics server for {targets.Length} targets...");
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
//var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
//var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
//if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
var workflow = tools.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
//return runningContainers;
return null!;
Log("Metrics server started.");
return runningContainers;
}
//private string GeneratePrometheusConfig(RunningContainer[] nodes)
//{
// var config = "";
// config += "global:\n";
// config += " scrape_interval: 10s\n";
// config += " scrape_timeout: 10s\n";
// config += "\n";
// config += "scrape_configs:\n";
// config += " - job_name: services\n";
// config += " metrics_path: /metrics\n";
// config += " static_configs:\n";
// config += " - targets:\n";
public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
{
var metricsQuery = new MetricsQuery(tools, metricsContainer);
return new MetricsAccess(metricsQuery, target);
}
// foreach (var node in nodes)
// {
// var ip = node.Pod.PodInfo.Ip;
// var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
// config += $" - '{ip}:{port}'\n";
// }
private void Log(string msg)
{
tools.GetLog().Log(msg);
}
// var bytes = Encoding.ASCII.GetBytes(config);
// return Convert.ToBase64String(bytes);
//}
private static string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
{
var config = "";
config += "global:\n";
config += " scrape_interval: 10s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var target in targets)
{
config += $" - '{target.Ip}:{target.Port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
}
}

22
MetricsPlugin/PrometheusStartupConfig.cs
View File

@ -1,12 +1,12 @@
//namespace DistTestCore.Metrics
//{
// public class PrometheusStartupConfig
// {
// public PrometheusStartupConfig(string prometheusConfigBase64)
// {
// PrometheusConfigBase64 = prometheusConfigBase64;
// }
namespace MetricsPlugin
{
public class PrometheusStartupConfig
{
public PrometheusStartupConfig(string prometheusConfigBase64)
{
PrometheusConfigBase64 = prometheusConfigBase64;
}
// public string PrometheusConfigBase64 { get; }
// }
//}
public string PrometheusConfigBase64 { get; }
}
}

3
Tests/BasicTests/ExampleTests.cs
View File

@ -1,5 +1,6 @@
using CodexPlugin;
using DistTestCore;
using MetricsPlugin;
using NUnit.Framework;
using Utils;
@ -23,6 +24,8 @@ namespace Tests.BasicTests
[Test]
public void TwoMetricsExample()
{
var rc = Ci.StartMetricsCollector();
//var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics());
//var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics());

1
Tests/Tests.csproj
View File

@ -15,6 +15,7 @@
<ItemGroup>
<ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />
</ItemGroup>
</Project>