codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from benbierens/feature/metrics 

Feature/metrics
This commit is contained in:
Ben Bierens 2023-03-31 10:56:33 +02:00 committed by GitHub
parent 8c16ae3385 6af0890571
commit d8ff9e4d02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 777 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CodexDistTestCore/CodexNodeContainer.cs
View File

@ -2,7 +2,7 @@
{
public class CodexNodeContainer
{
public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir)
public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort)
{
Name = name;
ServicePort = servicePort;
@ -12,6 +12,7 @@
DiscoveryPort = discoveryPort;
ListenPort = listenPort;
DataDir = dataDir;
MetricsPort = metricsPort;
}
public string Name { get; }
@ -22,6 +23,7 @@
public int DiscoveryPort { get; }
public int ListenPort { get; }
public string DataDir { get; }
public int MetricsPort { get; }
}
public class CodexGroupNumberSource
@ -57,7 +59,7 @@
this.groupContainerFactory = groupContainerFactory;
}
public CodexNodeContainer CreateNext()
public CodexNodeContainer CreateNext(OfflineCodexNodes offline)
{
var n = containerNameSource.GetNextNumber();
return new CodexNodeContainer(
@ -68,8 +70,15 @@
containerPortName: $"api-{n}",
discoveryPort: codexPortSource.GetNextNumber(),
listenPort: codexPortSource.GetNextNumber(),
dataDir: $"datadir{n}"
dataDir: $"datadir{n}",
metricsPort: GetMetricsPort(offline)
);
}
private int GetMetricsPort(OfflineCodexNodes offline)
{
if (offline.MetricsEnabled) return codexPortSource.GetNextNumber();
return 0;
}
}
}

7
CodexDistTestCore/CodexNodeLog.cs
View File

@ -2,7 +2,12 @@
namespace CodexDistTestCore
{
public class CodexNodeLog
public interface ICodexNodeLog
{
void AssertLogContains(string expectedString);
}
public class CodexNodeLog : ICodexNodeLog
{
private readonly LogFile logFile;

15
CodexDistTestCore/Config/CodexDockerImage.cs
View File

@ -25,12 +25,12 @@ namespace CodexDistTestCore.Config
{
public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
public void Create(OfflineCodexNodes node, CodexNodeContainer environment)
public void Create(OfflineCodexNodes node, CodexNodeContainer container)
{
AddVar("API_PORT", environment.ApiPort.ToString());
AddVar("DATA_DIR", environment.DataDir);
AddVar("DISC_PORT", environment.DiscoveryPort.ToString());
AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}");
AddVar("API_PORT", container.ApiPort.ToString());
AddVar("DATA_DIR", container.DataDir);
AddVar("DISC_PORT", container.DiscoveryPort.ToString());
AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}");
if (node.BootstrapNode != null)
{
@ -45,6 +45,11 @@ namespace CodexDistTestCore.Config
{
AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!);
}
if (node.MetricsEnabled)
{
AddVar("METRICS_ADDR", "0.0.0.0");
AddVar("METRICS_PORT", container.MetricsPort.ToString());
}
}
private void AddVar(string key, string value)

3
CodexDistTestCore/Config/K8sCluster.cs
View File

@ -17,7 +17,8 @@ namespace CodexDistTestCore.Config
public KubernetesClientConfiguration GetK8sClientConfig()
{
if (config != null) return config;
config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
//config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
config = KubernetesClientConfiguration.BuildDefaultConfig();
return config;
}

13
CodexDistTestCore/DistTest.cs
View File

@ -57,7 +57,7 @@ namespace CodexDistTestCore
try
{
log.EndTest();
IncludeLogsOnTestFailure();
IncludeLogsAndMetricsOnTestFailure();
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
@ -78,19 +78,20 @@ namespace CodexDistTestCore
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
private void IncludeLogsOnTestFailure()
private void IncludeLogsAndMetricsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
if (IsDownloadingLogsEnabled())
if (IsDownloadingLogsAndMetricsEnabled())
{
log.Log("Downloading all CodexNode logs because of test failure...");
log.Log("Downloading all CodexNode logs and metrics because of test failure...");
k8sManager.ForEachOnlineGroup(DownloadLogs);
k8sManager.DownloadAllMetrics();
}
else
{
log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute.");
log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
}
}
}
@ -105,7 +106,7 @@ namespace CodexDistTestCore
}
}
private bool IsDownloadingLogsEnabled()
private bool IsDownloadingLogsAndMetricsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);

30
CodexDistTestCore/FileManager.cs
View File

@ -80,28 +80,30 @@ namespace CodexDistTestCore
return info.Length;
}
public void AssertIsEqual(TestFile? other)
public void AssertIsEqual(TestFile? actual)
{
if (other == null) Assert.Fail("TestFile is null.");
if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
if (actual == null) Assert.Fail("TestFile is null.");
if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);
Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
var bytes1 = new byte[FileManager.ChunkSize];
var bytes2 = new byte[FileManager.ChunkSize];
using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
var read1 = 0;
var read2 = 0;
var bytesExpected = new byte[FileManager.ChunkSize];
var bytesActual = new byte[FileManager.ChunkSize];
var readExpected = 0;
var readActual = 0;
while (true)
{
read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize);
read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize);
readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
if (read1 == 0 && read2 == 0) return;
Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length.");
CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal.");
if (readExpected == 0 && readActual == 0) return;
Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
}
}
}

34
CodexDistTestCore/K8sManager.cs
View File

@ -14,11 +14,13 @@
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
private readonly MetricsAggregator metricsAggregator;
public K8sManager(TestLog log, IFileManager fileManager)
{
this.log = log;
this.fileManager = fileManager;
metricsAggregator = new MetricsAggregator(log, this);
}
public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
@ -29,6 +31,11 @@
log.Log($"{online.Describe()} online.");
if (offline.MetricsEnabled)
{
BringOnlineMetrics(online);
}
return online;
}
@ -58,20 +65,41 @@
K8s(k => k.FetchPodLog(node, logHandler));
}
public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
{
var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
PrometheusInfo? info = null;
K8s(k => info = k.BringOnlinePrometheus(spec));
return info!;
}
public void DownloadAllMetrics()
{
metricsAggregator.DownloadAllMetrics();
}
private void BringOnlineMetrics(CodexNodeGroup group)
{
var onlineNodes = group.Nodes.Cast<OnlineCodexNode>().ToArray();
metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline.NumberOfNodes);
var containers = CreateContainers(offline);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodeGroups.Add(result);
return result;
}
private CodexNodeContainer[] CreateContainers(int number)
private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline)
{
var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
var containers = new List<CodexNodeContainer>();
for (var i = 0; i < number; i++) containers.Add(factory.CreateNext());
for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline));
return containers.ToArray();
}

45
CodexDistTestCore/K8sOperations.cs
View File

@ -1,5 +1,6 @@
using CodexDistTestCore.Config;
using k8s;
using k8s.KubeConfigModels;
using k8s.Models;
using NUnit.Framework;
@ -57,7 +58,23 @@ namespace CodexDistTestCore
logHandler.Log(stream);
}
public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
{
EnsureTestNamespace();
CreatePrometheusDeployment(spec);
CreatePrometheusService(spec);
WaitUntilPrometheusOnline(spec);
return new PrometheusInfo(spec.ServicePort, FetchNewPod());
}
private void FetchPodInfo(CodexNodeGroup online)
{
online.PodInfo = FetchNewPod();
}
private PodInfo FetchNewPod()
{
var pods = client.ListNamespacedPod(K8sNamespace).Items;
@ -65,12 +82,13 @@ namespace CodexDistTestCore
Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
var newPod = newPods.Single();
online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP);
var info = new PodInfo(newPod.Name(), newPod.Status.PodIP);
Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure.");
Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure.");
Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure.");
Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure.");
knownPods.Add(newPod.Name());
return info;
}
#region Waiting
@ -103,6 +121,16 @@ namespace CodexDistTestCore
WaitUntil(() => !IsTestNamespaceOnline());
}
private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec)
{
var deploymentName = spec.GetDeploymentName();
WaitUntil(() =>
{
var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
private void WaitUntil(Func<bool> predicate)
{
var start = DateTime.UtcNow;
@ -166,6 +194,11 @@ namespace CodexDistTestCore
online.Service = null;
}
private void CreatePrometheusService(K8sPrometheusSpecs spec)
{
client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
}
#endregion
#region Deployment management
@ -232,6 +265,7 @@ namespace CodexDistTestCore
Env = dockerImage.CreateEnvironmentVariables(offline, container)
});
}
return result;
}
@ -242,6 +276,11 @@ namespace CodexDistTestCore
online.Deployment = null;
}
private void CreatePrometheusDeployment(K8sPrometheusSpecs spec)
{
client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace);
}
#endregion
#region Namespace management

122
CodexDistTestCore/K8sPrometheusSpecs.cs Normal file
View File

@ -0,0 +1,122 @@
using CodexDistTestCore.Config;
using k8s.Models;
namespace CodexDistTestCore
{
public class K8sPrometheusSpecs
{
public const string ContainerName = "dtest-prom";
public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
private const string portName = "prom-1";
private readonly string config;
public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
{
ServicePort = servicePort;
PrometheusNumber = prometheusNumber;
this.config = config;
}
public int ServicePort { get; }
public int PrometheusNumber { get; }
public string GetDeploymentName()
{
return "test-prom" + PrometheusNumber;
}
public V1Deployment CreatePrometheusDeployment()
{
var deploymentSpec = new V1Deployment
{
ApiVersion = "apps/v1",
Metadata = new V1ObjectMeta
{
Name = GetDeploymentName(),
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1DeploymentSpec
{
Replicas = 1,
Selector = new V1LabelSelector
{
MatchLabels = CreateSelector()
},
Template = new V1PodTemplateSpec
{
Metadata = new V1ObjectMeta
{
Labels = CreateSelector()
},
Spec = new V1PodSpec
{
Containers = new List<V1Container>
{
new V1Container
{
Name = ContainerName,
Image = dockerImage,
Ports = new List<V1ContainerPort>
{
new V1ContainerPort
{
ContainerPort = 9090,
Name = portName
}
},
Env = new List<V1EnvVar>
{
new V1EnvVar
{
Name = "PROM_CONFIG",
Value = config
}
}
}
}
}
}
}
};
return deploymentSpec;
}
public V1Service CreatePrometheusService()
{
var serviceSpec = new V1Service
{
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = "codex-prom-service" + PrometheusNumber,
NamespaceProperty = K8sCluster.K8sNamespace
},
Spec = new V1ServiceSpec
{
Type = "NodePort",
Selector = CreateSelector(),
Ports = new List<V1ServicePort>
{
new V1ServicePort
{
Name = "prom-service" + PrometheusNumber,
Protocol = "TCP",
Port = 9090,
TargetPort = portName,
NodePort = ServicePort
}
}
}
};
return serviceSpec;
}
private Dictionary<string, string> CreateSelector()
{
return new Dictionary<string, string> { { "test-prom", "dtest-prom" } };
}
}
}

63
CodexDistTestCore/MetricsAccess.cs Normal file
View File

@ -0,0 +1,63 @@
using NUnit.Framework;
using NUnit.Framework.Constraints;
namespace CodexDistTestCore
{
public interface IMetricsAccess
{
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
}
public class MetricsUnavailable : IMetricsAccess
{
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
throw new InvalidOperationException();
}
}
public class MetricsAccess : IMetricsAccess
{
private readonly MetricsQuery query;
private readonly OnlineCodexNode node;
public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
{
this.query = query;
this.node = node;
}
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
var metricSet = GetMetricWithTimeout(metricName, node);
var metricValue = metricSet.Values[0].Value;
Assert.That(metricValue, constraint, message);
}
private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
{
var start = DateTime.UtcNow;
while (true)
{
var mostRecent = GetMostRecent(metricName, node);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException();
}
Utils.Sleep(TimeSpan.FromSeconds(2));
}
}
private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
{
var result = query.GetMostRecent(metricName, node);
if (result == null) return null;
return result.Sets.LastOrDefault();
}
}
}

78
CodexDistTestCore/MetricsAggregator.cs Normal file
View File

@ -0,0 +1,78 @@
using NUnit.Framework;
using System.Text;
namespace CodexDistTestCore
{
public class MetricsAggregator
{
private readonly NumberSource prometheusNumberSource = new NumberSource(0);
private readonly TestLog log;
private readonly K8sManager k8sManager;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
public MetricsAggregator(TestLog log, K8sManager k8sManager)
{
this.log = log;
this.k8sManager = k8sManager;
}
public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
{
log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
var config = GeneratePrometheusConfig(nodes);
var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
var query = new MetricsQuery(prometheus);
activePrometheuses.Add(query, nodes);
log.Log("Metrics service started.");
foreach(var node in nodes)
{
node.Metrics = new MetricsAccess(query, node);
}
}
public void DownloadAllMetrics()
{
var download = new MetricsDownloader(log, activePrometheuses);
download.DownloadAllMetrics();
}
private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
{
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var node in nodes)
{
var ip = node.Group.PodInfo!.Ip;
var port = node.Container.MetricsPort;
config += $" - '{ip}:{port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
}
public class PrometheusInfo
{
public PrometheusInfo(int servicePort, PodInfo podInfo)
{
ServicePort = servicePort;
PodInfo = podInfo;
}
public int ServicePort { get; }
public PodInfo PodInfo { get; }
}
}

97
CodexDistTestCore/MetricsDownloader.cs Normal file
View File

@ -0,0 +1,97 @@
using System.Globalization;
namespace CodexDistTestCore
{
public class MetricsDownloader
{
private readonly TestLog log;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
{
this.log = log;
this.activePrometheuses = activePrometheuses;
}
public void DownloadAllMetrics()
{
foreach (var pair in activePrometheuses)
{
DownloadAllMetrics(pair.Key, pair.Value);
}
}
private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
{
foreach (var node in nodes)
{
DownloadAllMetricsForNode(query, node);
}
}
private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
{
var metrics = query.GetAllMetricsForNode(node);
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
var map = CreateValueMap(metrics);
WriteToFile(node.GetName(), headers, map);
}
private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
file.WriteRaw(string.Join(",", headers));
foreach (var pair in map)
{
file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
}
}
private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
{
var map = CreateForAllTimestamps(metrics);
foreach (var metric in metrics.Sets)
{
AddToMap(map, metric);
}
return map;
}
private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
{
var result = new Dictionary<DateTime, List<string>>();
var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
return result;
}
private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
{
foreach (var key in map.Keys)
{
map[key].Add(GetValueAtTimestamp(key, metric));
}
}
private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
{
var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
if (value == null) return "";
return value.Value.ToString(CultureInfo.InvariantCulture);
}
private string FormatTimestamp(DateTime key)
{
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
var diff = key - origin;
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
}
}
}

190
CodexDistTestCore/MetricsQuery.cs Normal file
View File

@ -0,0 +1,190 @@
using CodexDistTestCore.Config;
using System.Globalization;
namespace CodexDistTestCore
{
public class MetricsQuery
{
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Http http;
public MetricsQuery(PrometheusInfo prometheusInfo)
{
http = new Http(
k8sCluster.GetIp(),
prometheusInfo.ServicePort,
"api/v1");
}
public Metrics? GetMostRecent(string metricName, OnlineCodexNode node)
{
var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
if (response == null) return null;
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapSingleValue(r.value)
};
}).ToArray()
};
}
public Metrics? GetMetrics(string metricName)
{
var response = GetAll(metricName);
if (response == null) return null;
return MapResponseToMetrics(response);
}
public Metrics? GetAllMetricsForNode(OnlineCodexNode node)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return MapResponseToMetrics(response);
}
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
if (response.status != "success") return null;
return response;
}
private PrometheusQueryResponse? GetAll(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return response;
}
private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
{
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Name = r.metric.__name__,
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
private MetricsSetValue[] MapSingleValue(object[] value)
{
if (value != null && value.Length > 0)
{
return new[]
{
MapValue(value)
};
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue[] MapMultipleValues(object[][] values)
{
if (values != null && values.Length > 0)
{
return values.Select(v => MapValue(v)).ToArray();
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue MapValue(object[] value)
{
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
return new MetricsSetValue
{
Timestamp = ToTimestamp(value[0]),
Value = ToValue(value[1])
};
}
private string GetInstanceNameForNode(OnlineCodexNode node)
{
var pod = node.Group.PodInfo!;
return $"{pod.Ip}:{node.Container.MetricsPort}";
}
private string GetInstanceStringForNode(OnlineCodexNode node)
{
return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
}
private string GetQueryTimeRange()
{
return "[12h]";
}
private double ToValue(object v)
{
return Convert.ToDouble(v, CultureInfo.InvariantCulture);
}
private DateTime ToTimestamp(object v)
{
var unixSeconds = ToValue(v);
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
}
public class Metrics
{
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
}
public class MetricsSet
{
public string Name { get; set; } = string.Empty;
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
}
public class MetricsSetValue
{
public DateTime Timestamp { get; set; }
public double Value { get; set; }
}
public class PrometheusQueryResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryResponseData data { get; set; } = new();
}
public class PrometheusQueryResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
}
public class PrometheusQueryResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public object[] value { get; set; } = Array.Empty<object>();
public object[][] values { get; set; } = Array.Empty<object[]>();
}
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
public class PrometheusAllNamesResponse
{
public string status { get; set; } = string.Empty;
public string[] data { get; set; } = Array.Empty<string>();
}
}

9
CodexDistTestCore/OfflineCodexNodes.cs
View File

@ -6,6 +6,7 @@
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
IOfflineCodexNodes EnableMetrics();
ICodexNodeGroup BringOnline();
}
@ -34,12 +35,14 @@
public CodexLogLevel? LogLevel { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; private set; }
public ByteSize? StorageQuota { get; private set; }
public bool MetricsEnabled { get; private set; }
public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
{
this.k8SManager = k8SManager;
NumberOfNodes = numberOfNodes;
Location = Location.Unspecified;
MetricsEnabled = false;
}
public ICodexNodeGroup BringOnline()
@ -71,6 +74,12 @@
return this;
}
public IOfflineCodexNodes EnableMetrics()
{
MetricsEnabled = true;
return this;
}
public string Describe()
{
var args = string.Join(',', DescribeArgs());

6
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -9,7 +9,8 @@ namespace CodexDistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
CodexNodeLog DownloadLog();
ICodexNodeLog DownloadLog();
IMetricsAccess Metrics { get; }
}
public class OnlineCodexNode : IOnlineCodexNode
@ -30,6 +31,7 @@ namespace CodexDistTestCore
public CodexNodeContainer Container { get; }
public CodexNodeGroup Group { get; internal set; } = null!;
public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable();
public string GetName()
{
@ -80,7 +82,7 @@ namespace CodexDistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public CodexNodeLog DownloadLog()
public ICodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
}

4
CodexDistTestCore/PodLogDownloader.cs
View File

@ -8,9 +8,9 @@ namespace CodexDistTestCore
}
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
{
public DontDownloadLogsOnFailureAttribute()
public DontDownloadLogsAndMetricsOnFailureAttribute()
: base(Timing.UseLongTimeoutsKey)
{
}

47
CodexDistTestCore/TestLog.cs
View File

@ -39,11 +39,21 @@ namespace CodexDistTestCore
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
RenameLogFile();
}
}
public LogFile CreateSubfile()
private void RenameLogFile()
{
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}");
file.ConcatToFilename("_FAILED");
}
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
}
private static string GetTestName()
@ -63,10 +73,17 @@ namespace CodexDistTestCore
public class LogFile
{
private readonly DateTime now;
private string name;
private readonly string ext;
private readonly string filepath;
public LogFile(DateTime now, string name)
public LogFile(DateTime now, string name, string ext = "log")
{
this.now = now;
this.name = name;
this.ext = ext;
filepath = Path.Join(
LogConfig.LogRoot,
$"{now.Year}-{Pad(now.Month)}",
@ -74,12 +91,11 @@ namespace CodexDistTestCore
Directory.CreateDirectory(filepath);
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
GenerateFilename();
}
public string FullFilename { get; }
public string FilenameWithoutPath { get; }
public string FullFilename { get; private set; } = string.Empty;
public string FilenameWithoutPath { get; private set; } = string.Empty;
public void Write(string message)
{
@ -98,6 +114,17 @@ namespace CodexDistTestCore
}
}
public void ConcatToFilename(string toAdd)
{
var oldFullName = FullFilename;
name += toAdd;
GenerateFilename();
File.Move(oldFullName, FullFilename);
}
private static string Pad(int n)
{
return n.ToString().PadLeft(2, '0');
@ -107,5 +134,11 @@ namespace CodexDistTestCore
{
return $"[{DateTime.UtcNow.ToString("u")}]";
}
private void GenerateFilename()
{
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
}
}
}

16
CodexDistTestCore/Timing.cs
View File

@ -40,6 +40,11 @@ namespace CodexDistTestCore
return GetTimes().K8sOperationTimeout();
}
public static TimeSpan WaitForMetricTimeout()
{
return GetTimes().WaitForMetricTimeout();
}
private static ITimeSet GetTimes()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
@ -55,6 +60,7 @@ namespace CodexDistTestCore
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
}
public class DefaultTimeSet : ITimeSet
@ -83,6 +89,11 @@ namespace CodexDistTestCore
{
return TimeSpan.FromMinutes(5);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromSeconds(30);
}
}
public class LongTimeSet : ITimeSet
@ -111,5 +122,10 @@ namespace CodexDistTestCore
{
return TimeSpan.FromMinutes(15);
}
public TimeSpan WaitForMetricTimeout()
{
return TimeSpan.FromMinutes(5);
}
}
}

4
LongTests/BasicTests/TestInfraTests.cs
View File

@ -8,7 +8,9 @@ namespace LongTests.BasicTests
[Test, UseLongTimeouts]
public void TestInfraShouldHave1000AddressSpacesPerPod()
{
var group = SetupCodexNodes(1000).BringOnline();
var group = SetupCodexNodes(1000)
.EnableMetrics() // Increases use of port address space per node.
.BringOnline();
var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();

28
Tests/BasicTests/SimpleTests.cs
View File

@ -20,7 +20,7 @@ namespace Tests.BasicTests
Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
}
[Test, DontDownloadLogsOnFailure]
[Test, DontDownloadLogsAndMetricsOnFailure]
public void CanAccessLogs()
{
var node = SetupCodexNodes(1).BringOnline()[0];
@ -30,6 +30,31 @@ namespace Tests.BasicTests
log.AssertLogContains("Started codex node");
}
[Test]
public void TwoMetricsExample()
{
var group = SetupCodexNodes(2)
.EnableMetrics()
.BringOnline();
var group2 = SetupCodexNodes(2)
.EnableMetrics()
.BringOnline();
var primary = group[0];
var secondary = group[1];
var primary2 = group2[0];
var secondary2 = group2[1];
primary.ConnectToPeer(secondary);
primary2.ConnectToPeer(secondary2);
Thread.Sleep(TimeSpan.FromMinutes(5));
primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
}
[Test]
public void OneClientTest()
{
@ -66,6 +91,7 @@ namespace Tests.BasicTests
}
[Test]
[Ignore("Requires Location map to be configured for k8s cluster.")]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)