status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implements metrics plugin

This commit is contained in:
benbierens 2023-09-13 11:25:08 +02:00
parent a05a82c030
commit 1ca3ddc67e
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
17 changed files with 424 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Core/TimeSet.cs
View File

@ -7,7 +7,6 @@
TimeSpan HttpCallRetryDelay(); TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay(); TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout(); TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
} }
public class DefaultTimeSet : ITimeSet public class DefaultTimeSet : ITimeSet
@ -36,10 +35,5 @@
{ {
return TimeSpan.FromMinutes(30); return TimeSpan.FromMinutes(30);
} }
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromSeconds(30);
}
} }
} }

5
DistTestCore/LongTimeSet.cs
View File

@ -28,10 +28,5 @@ namespace DistTestCore
{ {
return TimeSpan.FromMinutes(15); return TimeSpan.FromMinutes(15);
} }
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromMinutes(5);
}
} }
} }

21
MetricsPlugin/CoreInterfaceExtensions.cs
View File

@ -1,13 +1,30 @@
using Core; using Core;
using KubernetesWorkflow; using KubernetesWorkflow;
using Logging;
namespace MetricsPlugin namespace MetricsPlugin
{ {
public static class CoreInterfaceExtensions public static class CoreInterfaceExtensions
{ {
public static RunningContainer StartMetricsCollector(this CoreInterface ci, RunningContainers[] scrapeTargets) public static RunningContainers StartMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
{ {
return null!;// Plugin(ci).StartMetricsCollector(scrapeTargets); return Plugin(ci).StartMetricsCollector(scrapeTargets);
}
public static IMetricsAccess GetMetricsFor(this CoreInterface ci, RunningContainers metricsContainer, IMetricsScrapeTarget scrapeTarget)
{
return Plugin(ci).CreateAccessForTarget(metricsContainer, scrapeTarget);
}
public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
{
var rc = ci.StartMetricsCollector(scrapeTargets);
return scrapeTargets.Select(t => ci.GetMetricsFor(rc, t)).ToArray();
}
public static LogFile? DownloadAllMetrics(this CoreInterface ci, IMetricsAccess metricsAccess, string targetName)
{
return Plugin(ci).DownloadAllMetrics(metricsAccess, targetName);
} }
private static MetricsPlugin Plugin(CoreInterface ci) private static MetricsPlugin Plugin(CoreInterface ci)

25
MetricsPlugin/GrafanaContainerRecipe.cs
View File

@ -1,25 +0,0 @@
//using KubernetesWorkflow;
//namespace DistTestCore.Metrics
//{
// public class GrafanaContainerRecipe : DefaultContainerRecipe
// {
// public override string AppName => "grafana";
// public override string Image => "grafana/grafana-oss:10.0.3";
// public const string DefaultAdminUser = "adminium";
// public const string DefaultAdminPassword = "passwordium";
// protected override void InitializeRecipe(StartupConfig startupConfig)
// {
// AddExposedPort(3000);
// AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
// AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
// AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
// AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
// }
// }
//}

188
MetricsPlugin/GrafanaStarter.cs
View File

@ -1,188 +0,0 @@
using KubernetesWorkflow;
namespace MetricsPlugin
{
public class GrafanaStarter
{
private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
public GrafanaStarter()
{
}
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)//, CodexSetup codexSetup)
{
return null!;
//LogStart($"Starting dashboard server");
//var grafanaContainer = StartGrafanaContainer();
//var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
//var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
//Log("Connecting datasource...");
//AddDataSource(http, prometheusContainer);
//Log("Uploading dashboard configurations...");
//var jsons = ReadEachDashboardJsonFile(codexSetup);
//var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
//LogEnd("Dashboard server started.");
//return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
}
//private RunningContainer StartGrafanaContainer()
//{
// var startupConfig = new StartupConfig();
// var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
// var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
// if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
// return grafanaContainers.Containers.First();
//}
//private void AddBasicAuth(HttpClient client)
//{
// client.SetBasicAuthentication(
// GrafanaContainerRecipe.DefaultAdminUser,
// GrafanaContainerRecipe.DefaultAdminPassword);
//}
//private static void AddDataSource(Http http, RunningContainer prometheusContainer)
//{
// var prometheusAddress = prometheusContainer.ClusterExternalAddress;
// var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
// var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
// {
// uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
// name = "CodexPrometheus",
// type = "prometheus",
// url = prometheusUrl,
// access = "proxy",
// basicAuth = false,
// jsonData = new GrafanaDataSourceJsonData
// {
// httpMethod = "POST"
// }
// });
// if (response.message != "Datasource added")
// {
// throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
// }
//}
//public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
//{
// var request = GetDashboardCreateRequest(dashboardJson);
// var response = http.HttpPostString("dashboards/db", request);
// var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
// if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
// var grafanaAddress = grafanaContainer.ClusterExternalAddress;
// return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
//}
//private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
//{
// var assembly = Assembly.GetExecutingAssembly();
// var resourceNames = new[]
// {
// "DistTestCore.Metrics.dashboard.json"
// };
// return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
//}
//private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
//{
// using var stream = assembly.GetManifestResourceStream(resourceName);
// if (stream == null) throw new Exception("Unable to find resource " + resourceName);
// using var reader = new StreamReader(stream);
// return ApplyReplacements(reader.ReadToEnd(), codexSetup);
//}
//private static string ApplyReplacements(string input, CodexSetup codexSetup)
//{
// var quotaString = GetQuotaString(codexSetup);
// var softMaxString = GetSoftMaxString(codexSetup);
// return input
// .Replace(StorageQuotaThresholdReplaceToken, quotaString)
// .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
//}
//private static string GetQuotaString(CodexSetup codexSetup)
//{
// return GetCodexStorageQuotaInBytes(codexSetup).ToString();
//}
//private static string GetSoftMaxString(CodexSetup codexSetup)
//{
// var quota = GetCodexStorageQuotaInBytes(codexSetup);
// var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
// return softMax.ToString();
//}
//private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
//{
// if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
// // Codex default: 8GB
// return 8.GB().SizeInBytes;
//}
//private static string GetDashboardCreateRequest(string dashboardJson)
//{
// return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
//}
}
public class GrafanaStartInfo
{
public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
{
DashboardUrls = dashboardUrls;
Container = container;
}
public string[] DashboardUrls { get; }
public RunningContainer Container { get; }
}
public class GrafanaDataSourceRequest
{
public string uid { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
public string type { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public string access { get; set; } = string.Empty;
public bool basicAuth { get; set; }
public GrafanaDataSourceJsonData jsonData { get; set; } = new();
}
public class GrafanaDataSourceResponse
{
public int id { get; set; }
public string message { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
}
public class GrafanaDataSourceJsonData
{
public string httpMethod { get; set; } = string.Empty;
}
public class GrafanaPostDashboardResponse
{
public int id { get; set; }
public string slug { get; set; } = string.Empty;
public string status { get; set; } = string.Empty;
public string uid { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public int version { get; set; }
}
}

128
MetricsPlugin/MetricsAccess.cs
View File

@ -1,81 +1,69 @@
//using DistTestCore.Helpers; using Utils;
//using KubernetesWorkflow;
//using Logging;
//using NUnit.Framework;
//using NUnit.Framework.Constraints;
//using Utils;
//namespace DistTestCore.Metrics namespace MetricsPlugin
//{ {
// public interface IMetricsAccess public interface IMetricsAccess
// { {
// void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); Metrics? GetAllMetrics();
// } MetricsSet GetMetric(string metricName);
MetricsSet GetMetric(string metricName, TimeSpan timeout);
}
// public class MetricsAccess : IMetricsAccess public class MetricsAccess : IMetricsAccess
// { {
// private readonly BaseLog log; private readonly MetricsQuery query;
// private readonly ITimeSet timeSet; private readonly IMetricsScrapeTarget target;
// private readonly MetricsQuery query;
// private readonly RunningContainer node;
// public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) public MetricsAccess(MetricsQuery query, IMetricsScrapeTarget target)
// { {
// this.log = log; this.query = query;
// this.timeSet = timeSet; this.target = target;
// this.query = query; }
// this.node = node;
// }
// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") //public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
// { //{
// AssertHelpers.RetryAssert(constraint, () => // AssertHelpers.RetryAssert(constraint, () =>
// { // {
// var metricSet = GetMetricWithTimeout(metricName); // var metricSet = GetMetricWithTimeout(metricName);
// var metricValue = metricSet.Values[0].Value; // var metricValue = metricSet.Values[0].Value;
// log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); // log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
// return metricValue; // return metricValue;
// }, message); // }, message);
// } //}
// public Metrics? GetAllMetrics() public Metrics? GetAllMetrics()
// { {
// return query.GetAllMetricsForNode(node); return query.GetAllMetricsForNode(target);
// } }
// private MetricsSet GetMetricWithTimeout(string metricName) public MetricsSet GetMetric(string metricName)
// { {
// var start = DateTime.UtcNow; return GetMetric(metricName, TimeSpan.FromSeconds(10));
}
// while (true) public MetricsSet GetMetric(string metricName, TimeSpan timeout)
// { {
// var mostRecent = GetMostRecent(metricName); var start = DateTime.UtcNow;
// if (mostRecent != null) return mostRecent;
// if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
// {
// Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
// throw new TimeoutException();
// }
// Time.Sleep(TimeSpan.FromSeconds(2)); while (true)
// } {
// } var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException();
}
// private MetricsSet? GetMostRecent(string metricName) Time.Sleep(TimeSpan.FromSeconds(2));
// { }
// var result = query.GetMostRecent(metricName, node); }
// if (result == null) return null;
// return result.Sets.LastOrDefault();
// }
// }
// public class MetricsUnavailable : IMetricsAccess private MetricsSet? GetMostRecent(string metricName)
// { {
// public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") var result = query.GetMostRecent(metricName, target);
// { if (result == null) return null;
// Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); return result.Sets.LastOrDefault();
// throw new InvalidOperationException(); }
// } }
// } }
//}

35
MetricsPlugin/MetricsAccessFactory.cs
View File

@ -1,35 +0,0 @@
//using KubernetesWorkflow;
//namespace DistTestCore.Metrics
//{
// public interface IMetricsAccessFactory
// {
// IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
// }
// public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
// {
// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
// {
// return new MetricsUnavailable();
// }
// }
// public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
// {
// private readonly TestLifecycle lifecycle;
// private readonly RunningContainers prometheusContainer;
// public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
// {
// this.lifecycle = lifecycle;
// this.prometheusContainer = prometheusContainer;
// }
// public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
// {
// var query = new MetricsQuery(lifecycle, prometheusContainer);
// return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
// }
// }
//}

134
MetricsPlugin/MetricsDownloader.cs
View File

@ -1,80 +1,82 @@
//using Logging; using Logging;
//using System.Globalization; using System.Globalization;
//namespace DistTestCore.Metrics namespace MetricsPlugin
//{ {
// public class MetricsDownloader public class MetricsDownloader
// { {
// private readonly BaseLog log; private readonly ILog log;
// public MetricsDownloader(BaseLog log) public MetricsDownloader(ILog log)
// { {
// this.log = log; this.log = log;
// } }
// public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access) public LogFile? DownloadAllMetrics(string targetName, IMetricsAccess access)
// { {
// var metrics = access.GetAllMetrics(); var metrics = access.GetAllMetrics();
// if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return null;
// var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
// var map = CreateValueMap(metrics); var map = CreateValueMap(metrics);
// WriteToFile(nodeName, headers, map); return WriteToFile(targetName, headers, map);
// } }
// private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map) private LogFile WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
// { {
// var file = log.CreateSubfile("csv"); var file = log.CreateSubfile("csv");
// log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}"); log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
// file.WriteRaw(string.Join(",", headers)); file.WriteRaw(string.Join(",", headers));
// foreach (var pair in map) foreach (var pair in map)
// { {
// file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
// } }
// }
// private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics) return file;
// { }
// var map = CreateForAllTimestamps(metrics);
// foreach (var metric in metrics.Sets)
// {
// AddToMap(map, metric);
// }
// return map;
// } private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
{
var map = CreateForAllTimestamps(metrics);
foreach (var metric in metrics.Sets)
{
AddToMap(map, metric);
}
return map;
// private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics) }
// {
// var result = new Dictionary<DateTime, List<string>>();
// var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
// foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
// return result;
// }
// private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric) private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
// { {
// foreach (var key in map.Keys) var result = new Dictionary<DateTime, List<string>>();
// { var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
// map[key].Add(GetValueAtTimestamp(key, metric)); foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
// } return result;
// } }
// private string GetValueAtTimestamp(DateTime key, MetricsSet metric) private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
// { {
// var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); foreach (var key in map.Keys)
// if (value == null) return ""; {
// return value.Value.ToString(CultureInfo.InvariantCulture); map[key].Add(GetValueAtTimestamp(key, metric));
// } }
}
// private string FormatTimestamp(DateTime key) private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
// { {
// var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
// var diff = key - origin; if (value == null) return "";
// return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); return value.Value.ToString(CultureInfo.InvariantCulture);
// } }
// }
//} private string FormatTimestamp(DateTime key)
{
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
var diff = key - origin;
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
}
}
}

9
MetricsPlugin/MetricsMode.cs
View File

@ -1,9 +0,0 @@
//namespace DistTestCore.Metrics
//{
// public enum MetricsMode
// {
// None,
// Record,
// Dashboard
// }
//}

17
MetricsPlugin/MetricsPlugin.cs
View File

@ -1,5 +1,6 @@
using Core; using Core;
using KubernetesWorkflow; using KubernetesWorkflow;
using Logging;
namespace MetricsPlugin namespace MetricsPlugin
{ {
@ -14,19 +15,29 @@ namespace MetricsPlugin
starter = new PrometheusStarter(tools); starter = new PrometheusStarter(tools);
} }
public void Announce() public void Announce()
{ {
//log.Log("Hi from the metrics plugin."); tools.GetLog().Log("Hi from the metrics plugin.");
} }
public void Decommission() public void Decommission()
{ {
} }
public RunningContainers StartMetricsCollector(RunningContainers[] scrapeTargets) public RunningContainers StartMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
{ {
return starter.CollectMetricsFor(scrapeTargets); return starter.CollectMetricsFor(scrapeTargets);
} }
public MetricsAccess CreateAccessForTarget(RunningContainers runningContainers, IMetricsScrapeTarget target)
{
return starter.CreateAccessForTarget(runningContainers, target);
}
public LogFile? DownloadAllMetrics(IMetricsAccess metricsAccess, string targetName)
{
var downloader = new MetricsDownloader(tools.GetLog());
return downloader.DownloadAllMetrics(targetName, metricsAccess);
}
} }
} }

333
MetricsPlugin/MetricsQuery.cs
View File

@ -1,198 +1,189 @@
//using DistTestCore.Codex; using Core;
//using KubernetesWorkflow; using KubernetesWorkflow;
//using System.Globalization; using System.Globalization;
//namespace DistTestCore.Metrics namespace MetricsPlugin
//{ {
// public class MetricsQuery public class MetricsQuery
// { {
// private readonly Http http; private readonly Http http;
// public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers) public MetricsQuery(IPluginTools tools, RunningContainers runningContainers)
// { {
// RunningContainers = runningContainers; RunningContainers = runningContainers;
http = tools.CreateHttp(runningContainers.Containers[0].Address, "api/v1");
}
// var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]); public RunningContainers RunningContainers { get; }
// http = new Http( public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
// lifecycle.Log, {
// lifecycle.TimeSet, var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
// address, if (response == null) return null;
// "api/v1");
// }
// public RunningContainers RunningContainers { get; } return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapSingleValue(r.value)
};
}).ToArray()
};
}
// public Metrics? GetMostRecent(string metricName, RunningContainer node) public Metrics? GetMetrics(string metricName)
// { {
// var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); var response = GetAll(metricName);
// if (response == null) return null; if (response == null) return null;
return MapResponseToMetrics(response);
}
// return new Metrics public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
// { {
// Sets = response.data.result.Select(r => var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
// { if (response.status != "success") return null;
// return new MetricsSet return MapResponseToMetrics(response);
// { }
// Instance = r.metric.instance,
// Values = MapSingleValue(r.value)
// };
// }).ToArray()
// };
// }
// public Metrics? GetMetrics(string metricName) private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
// { {
// var response = GetAll(metricName); var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
// if (response == null) return null; if (response.status != "success") return null;
// return MapResponseToMetrics(response); return response;
// } }
// public Metrics? GetAllMetricsForNode(RunningContainer node) private PrometheusQueryResponse? GetAll(string metricName)
// { {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
// if (response.status != "success") return null; if (response.status != "success") return null;
// return MapResponseToMetrics(response); return response;
// } }
// private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
// { {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); return new Metrics
// if (response.status != "success") return null; {
// return response; Sets = response.data.result.Select(r =>
// } {
return new MetricsSet
{
Name = r.metric.__name__,
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
// private PrometheusQueryResponse? GetAll(string metricName) private MetricsSetValue[] MapSingleValue(object[] value)
// { {
// var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}"); if (value != null && value.Length > 0)
// if (response.status != "success") return null; {
// return response; return new[]
// } {
MapValue(value)
};
}
return Array.Empty<MetricsSetValue>();
}
// private Metrics MapResponseToMetrics(PrometheusQueryResponse response) private MetricsSetValue[] MapMultipleValues(object[][] values)
// { {
// return new Metrics if (values != null && values.Length > 0)
// { {
// Sets = response.data.result.Select(r => return values.Select(v => MapValue(v)).ToArray();
// { }
// return new MetricsSet return Array.Empty<MetricsSetValue>();
// { }
// Name = r.metric.__name__,
// Instance = r.metric.instance,
// Values = MapMultipleValues(r.values)
// };
// }).ToArray()
// };
// }
// private MetricsSetValue[] MapSingleValue(object[] value) private MetricsSetValue MapValue(object[] value)
// { {
// if (value != null && value.Length > 0) if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
// {
// return new[]
// {
// MapValue(value)
// };
// }
// return Array.Empty<MetricsSetValue>();
// }
// private MetricsSetValue[] MapMultipleValues(object[][] values) return new MetricsSetValue
// { {
// if (values != null && values.Length > 0) Timestamp = ToTimestamp(value[0]),
// { Value = ToValue(value[1])
// return values.Select(v => MapValue(v)).ToArray(); };
// } }
// return Array.Empty<MetricsSetValue>();
// }
// private MetricsSetValue MapValue(object[] value) private string GetInstanceNameForNode(IMetricsScrapeTarget target)
// { {
// if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); return $"{target.Ip}:{target.Port}";
}
// return new MetricsSetValue private string GetInstanceStringForNode(IMetricsScrapeTarget target)
// { {
// Timestamp = ToTimestamp(value[0]), return "{instance=\"" + GetInstanceNameForNode(target) + "\"}";
// Value = ToValue(value[1]) }
// };
// }
// private string GetInstanceNameForNode(RunningContainer node) private string GetQueryTimeRange()
// { {
// var ip = node.Pod.PodInfo.Ip; return "[12h]";
// var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; }
// return $"{ip}:{port}";
// }
// private string GetInstanceStringForNode(RunningContainer node) private double ToValue(object v)
// { {
// return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; return Convert.ToDouble(v, CultureInfo.InvariantCulture);
// } }
// private string GetQueryTimeRange() private DateTime ToTimestamp(object v)
// { {
// return "[12h]"; var unixSeconds = ToValue(v);
// } return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
}
// private double ToValue(object v) public class Metrics
// { {
// return Convert.ToDouble(v, CultureInfo.InvariantCulture); public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
// } }
// private DateTime ToTimestamp(object v) public class MetricsSet
// { {
// var unixSeconds = ToValue(v); public string Name { get; set; } = string.Empty;
// return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); public string Instance { get; set; } = string.Empty;
// } public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
// } }
// public class Metrics public class MetricsSetValue
// { {
// public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>(); public DateTime Timestamp { get; set; }
// } public double Value { get; set; }
}
// public class MetricsSet public class PrometheusQueryResponse
// { {
// public string Name { get; set; } = string.Empty; public string status { get; set; } = string.Empty;
// public string Instance { get; set; } = string.Empty; public PrometheusQueryResponseData data { get; set; } = new();
// public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>(); }
// }
// public class MetricsSetValue public class PrometheusQueryResponseData
// { {
// public DateTime Timestamp { get; set; } public string resultType { get; set; } = string.Empty;
// public double Value { get; set; } public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
// } }
// public class PrometheusQueryResponse public class PrometheusQueryResponseDataResultEntry
// { {
// public string status { get; set; } = string.Empty; public ResultEntryMetric metric { get; set; } = new();
// public PrometheusQueryResponseData data { get; set; } = new(); public object[] value { get; set; } = Array.Empty<object>();
// } public object[][] values { get; set; } = Array.Empty<object[]>();
}
// public class PrometheusQueryResponseData public class ResultEntryMetric
// { {
// public string resultType { get; set; } = string.Empty; public string __name__ { get; set; } = string.Empty;
// public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>(); public string instance { get; set; } = string.Empty;
// } public string job { get; set; } = string.Empty;
}
// public class PrometheusQueryResponseDataResultEntry public class PrometheusAllNamesResponse
// { {
// public ResultEntryMetric metric { get; set; } = new(); public string status { get; set; } = string.Empty;
// public object[] value { get; set; } = Array.Empty<object>(); public string[] data { get; set; } = Array.Empty<string>();
// public object[][] values { get; set; } = Array.Empty<object[]>(); }
// } }
// public class ResultEntryMetric
// {
// public string __name__ { get; set; } = string.Empty;
// public string instance { get; set; } = string.Empty;
// public string job { get; set; } = string.Empty;
// }
// public class PrometheusAllNamesResponse
// {
// public string status { get; set; } = string.Empty;
// public string[] data { get; set; } = Array.Empty<string>();
// }
//}

32
MetricsPlugin/MetricsScrapeTarget.cs Normal file
View File

@ -0,0 +1,32 @@
using KubernetesWorkflow;
namespace MetricsPlugin
{
public interface IMetricsScrapeTarget
{
string Ip { get; }
int Port { get; }
}
public class MetricsScrapeTarget : IMetricsScrapeTarget
{
public MetricsScrapeTarget(string ip, int port)
{
Ip = ip;
Port = port;
}
public MetricsScrapeTarget(RunningContainer container, int port)
: this(container.Pod.PodInfo.Ip, port)
{
}
public MetricsScrapeTarget(RunningContainer container, Port port)
: this(container, port.Number)
{
}
public string Ip { get; }
public int Port { get; }
}
}

31
MetricsPlugin/PrometheusContainerRecipe.cs
View File

@ -1,18 +1,19 @@
//using KubernetesWorkflow; using Core;
using KubernetesWorkflow;
//namespace DistTestCore.Metrics namespace MetricsPlugin
//{ {
// public class PrometheusContainerRecipe : DefaultContainerRecipe public class PrometheusContainerRecipe : DefaultContainerRecipe
// { {
// public override string AppName => "prometheus"; public override string AppName => "prometheus";
// public override string Image => "codexstorage/dist-tests-prometheus:latest"; public override string Image => "codexstorage/dist-tests-prometheus:latest";
// protected override void InitializeRecipe(StartupConfig startupConfig) protected override void InitializeRecipe(StartupConfig startupConfig)
// { {
// var config = startupConfig.Get<PrometheusStartupConfig>(); var config = startupConfig.Get<PrometheusStartupConfig>();
// AddExposedPortAndVar("PROM_PORT"); AddExposedPortAndVar("PROM_PORT");
// AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
// } }
// } }
//} }

70
MetricsPlugin/PrometheusStarter.cs
View File

@ -1,5 +1,6 @@
using Core; using Core;
using KubernetesWorkflow; using KubernetesWorkflow;
using System.Text;
namespace MetricsPlugin namespace MetricsPlugin
{ {
@ -12,42 +13,51 @@ namespace MetricsPlugin
this.tools = tools; this.tools = tools;
} }
public RunningContainers CollectMetricsFor(RunningContainers[] containers) public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
{ {
//LogStart($"Starting metrics server for {containers.Describe()}"); Log($"Starting metrics server for {targets.Length} targets...");
//var startupConfig = new StartupConfig(); var startupConfig = new StartupConfig();
//startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
//var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var workflow = tools.CreateWorkflow();
//var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
//if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
//return runningContainers; Log("Metrics server started.");
return null!; return runningContainers;
} }
//private string GeneratePrometheusConfig(RunningContainer[] nodes) public MetricsAccess CreateAccessForTarget(RunningContainers metricsContainer, IMetricsScrapeTarget target)
//{ {
// var config = ""; var metricsQuery = new MetricsQuery(tools, metricsContainer);
// config += "global:\n"; return new MetricsAccess(metricsQuery, target);
// config += " scrape_interval: 10s\n"; }
// config += " scrape_timeout: 10s\n";
// config += "\n";
// config += "scrape_configs:\n";
// config += " - job_name: services\n";
// config += " metrics_path: /metrics\n";
// config += " static_configs:\n";
// config += " - targets:\n";
// foreach (var node in nodes) private void Log(string msg)
// { {
// var ip = node.Pod.PodInfo.Ip; tools.GetLog().Log(msg);
// var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; }
// config += $" - '{ip}:{port}'\n";
// }
// var bytes = Encoding.ASCII.GetBytes(config); private static string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
// return Convert.ToBase64String(bytes); {
//} var config = "";
config += "global:\n";
config += " scrape_interval: 10s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var target in targets)
{
config += $" - '{target.Ip}:{target.Port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
} }
} }

22
MetricsPlugin/PrometheusStartupConfig.cs
View File

@ -1,12 +1,12 @@
//namespace DistTestCore.Metrics namespace MetricsPlugin
//{ {
// public class PrometheusStartupConfig public class PrometheusStartupConfig
// { {
// public PrometheusStartupConfig(string prometheusConfigBase64) public PrometheusStartupConfig(string prometheusConfigBase64)
// { {
// PrometheusConfigBase64 = prometheusConfigBase64; PrometheusConfigBase64 = prometheusConfigBase64;
// } }
// public string PrometheusConfigBase64 { get; } public string PrometheusConfigBase64 { get; }
// } }
//} }

3
Tests/BasicTests/ExampleTests.cs
View File

@ -1,5 +1,6 @@
using CodexPlugin; using CodexPlugin;
using DistTestCore; using DistTestCore;
using MetricsPlugin;
using NUnit.Framework; using NUnit.Framework;
using Utils; using Utils;
@ -23,6 +24,8 @@ namespace Tests.BasicTests
[Test] [Test]
public void TwoMetricsExample() public void TwoMetricsExample()
{ {
var rc = Ci.StartMetricsCollector();
//var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); //var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics());
//var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); //var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics());

1
Tests/Tests.csproj
View File

@ -15,6 +15,7 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" /> <ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" /> <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
<ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>