codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Using customized prometheus image to control configuration. This works.

This commit is contained in:
benbierens 2023-03-27 16:24:04 +02:00
parent da68fa1de8
commit 3d1ee8ebdb
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
9 changed files with 118 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CodexDistTestCore/CodexDistTestCore.csproj
View File

@ -12,7 +12,6 @@
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
<PackageReference Include="SharpZipLib" Version="1.4.2" />
</ItemGroup>
</Project>

11
CodexDistTestCore/DistTest.cs
View File

@ -87,11 +87,13 @@ namespace CodexDistTestCore
public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes)
{
Assert.That(nodes.All(n => HasMetricsEnable(n)),
var onlineNodes = nodes.Cast<OnlineCodexNode>().ToArray();
Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled),
"Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " +
"To use metrics, please use 'EnableMetrics()' when setting up Codex nodes.");
return metricsAggregator.BeginCollectingMetricsFor(nodes);
return metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
}
public void AssertWithTimeout<T>(Func<T> operation, T isEqualTo, string message)
@ -150,11 +152,6 @@ namespace CodexDistTestCore
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
}
private bool HasMetricsEnable(IOnlineCodexNode n)
{
return ((OnlineCodexNode)n).Group.Origin.MetricsEnabled;
}
}
public static class GlobalTestFailure

79
CodexDistTestCore/K8sCp.cs
View File

@ -1,79 +0,0 @@
using ICSharpCode.SharpZipLib.Tar;
using k8s;
using System.Text;
namespace CodexDistTestCore
{
// From: https://github.com/kubernetes-client/csharp/blob/master/examples/cp/Cp.cs
public class K8sCp
{
private readonly Kubernetes client;
public K8sCp(Kubernetes client)
{
this.client = client;
}
public async Task<int> CopyFileToPodAsync(string podName, string @namespace, string containerName, Stream inputFileStream, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken))
{
var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) =>
{
var fileInfo = new FileInfo(destinationFilePath);
try
{
using (var memoryStream = new MemoryStream())
{
using (var tarOutputStream = new TarOutputStream(memoryStream, Encoding.Default))
{
tarOutputStream.IsStreamOwner = false;
var fileSize = inputFileStream.Length;
var entry = TarEntry.CreateTarEntry(fileInfo.Name);
entry.Size = fileSize;
tarOutputStream.PutNextEntry(entry);
await inputFileStream.CopyToAsync(tarOutputStream);
tarOutputStream.CloseEntry();
}
memoryStream.Position = 0;
await memoryStream.CopyToAsync(stdIn);
await stdIn.FlushAsync();
}
}
catch (Exception ex)
{
throw new IOException($"Copy command failed: {ex.Message}");
}
using StreamReader streamReader = new StreamReader(stdError);
while (streamReader.EndOfStream == false)
{
string error = await streamReader.ReadToEndAsync();
throw new IOException($"Copy command failed: {error}");
}
});
string destinationFolder = GetFolderName(destinationFilePath);
return await client.NamespacedPodExecAsync(
podName,
@namespace,
containerName,
new string[] { "sh", "-c", $"tar xmf - -C {destinationFolder}" },
false,
handler,
cancellationToken);
}
private static string GetFolderName(string filePath)
{
var folderName = Path.GetDirectoryName(filePath);
return string.IsNullOrEmpty(folderName) ? "." : folderName;
}
}
}

13
CodexDistTestCore/K8sManager.cs
View File

@ -58,16 +58,13 @@
K8s(k => k.FetchPodLog(node, logHandler));
}
public PrometheusInfo BringOnlinePrometheus()
public PrometheusInfo BringOnlinePrometheus(string config)
{
PrometheusInfo? info = null;
K8s(k => info = k.BringOnlinePrometheus(codexGroupNumberSource.GetNextServicePort()));
return info!;
}
var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config);
public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath)
{
K8s(k => k.UploadFileToPod(podName, containerName, fileStream, destinationPath));
PrometheusInfo? info = null;
K8s(k => info = k.BringOnlinePrometheus(spec));
return info!;
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)

17
CodexDistTestCore/K8sOperations.cs
View File

@ -58,22 +58,15 @@ namespace CodexDistTestCore
logHandler.Log(stream);
}
public PrometheusInfo BringOnlinePrometheus(int servicePort)
public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
{
EnsureTestNamespace();
var spec = new K8sPrometheusSpecs();
CreatePrometheusDeployment(spec);
CreatePrometheusService(spec, servicePort);
CreatePrometheusService(spec);
WaitUntilPrometheusOnline(spec);
return new PrometheusInfo(servicePort, FetchNewPod());
}
public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath)
{
var cp = new K8sCp(client);
Utils.Wait(cp.CopyFileToPodAsync(podName, K8sCluster.K8sNamespace, containerName, fileStream, destinationPath));
return new PrometheusInfo(spec.ServicePort, FetchNewPod());
}
private void FetchPodInfo(CodexNodeGroup online)
@ -201,9 +194,9 @@ namespace CodexDistTestCore
online.Service = null;
}
private void CreatePrometheusService(K8sPrometheusSpecs spec, int servicePort)
private void CreatePrometheusService(K8sPrometheusSpecs spec)
{
client.CreateNamespacedService(spec.CreatePrometheusService(servicePort), K8sNamespace);
client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
}
#endregion

25
CodexDistTestCore/K8sPrometheusSpecs.cs
View File

@ -7,8 +7,17 @@ namespace CodexDistTestCore
{
public const string ContainerName = "dtest-prom";
public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
private const string dockerImage = "prom/prometheus:v2.30.3";
private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
private const string portName = "prom-1";
private readonly string config;
public K8sPrometheusSpecs(int servicePort, string config)
{
ServicePort = servicePort;
this.config = config;
}
public int ServicePort { get; }
public string GetDeploymentName()
{
@ -54,10 +63,14 @@ namespace CodexDistTestCore
Name = portName
}
},
Command = new List<string>
Env = new List<V1EnvVar>
{
$"--web.enable-lifecycle --config.file={ConfigFilepath}"
},
new V1EnvVar
{
Name = "PROM_CONFIG",
Value = config
}
}
}
}
}
@ -68,7 +81,7 @@ namespace CodexDistTestCore
return deploymentSpec;
}
public V1Service CreatePrometheusService(int servicePort)
public V1Service CreatePrometheusService()
{
var serviceSpec = new V1Service
{
@ -90,7 +103,7 @@ namespace CodexDistTestCore
Protocol = "TCP",
Port = 9090,
TargetPort = portName,
NodePort = servicePort
NodePort = ServicePort
}
}
}

55
CodexDistTestCore/MetricsAccess.cs
View File

@ -1,4 +1,6 @@
namespace CodexDistTestCore
using CodexDistTestCore.Config;
namespace CodexDistTestCore
{
public interface IMetricsAccess
{
@ -7,9 +9,60 @@
public class MetricsAccess : IMetricsAccess
{
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Http http;
public MetricsAccess(PrometheusInfo prometheusInfo)
{
http = new Http(
k8sCluster.GetIp(),
prometheusInfo.ServicePort,
"api/v1");
}
public int GetMostRecentInt(string metricName, IOnlineCodexNode node)
{
var now = DateTime.UtcNow;
var off = new DateTimeOffset(now);
var nowUnix = off.ToUnixTimeSeconds();
var hour = now.AddHours(-1);
var off2 = new DateTimeOffset(hour);
var hourUnix = off2.ToUnixTimeSeconds();
var response = http.HttpGetJson<PrometheusQueryRangeResponse>($"query_range?query=libp2p_peers&start={hourUnix}&end={nowUnix}&step=100");
return 0;
}
}
public class PrometheusQueryRangeResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryRangeResponseData data { get; set; } = new();
}
public class PrometheusQueryRangeResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryRangeResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryRangeResponseDataResultEntry>();
}
public class PrometheusQueryRangeResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public ResultEntryValue[] values { get; set; } = Array.Empty<ResultEntryValue>();
}
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
public class ResultEntryValue
{
}
}

76
CodexDistTestCore/MetricsAggregator.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework;
using System.Text;
namespace CodexDistTestCore
{
@ -6,7 +7,6 @@ namespace CodexDistTestCore
{
private readonly TestLog log;
private readonly K8sManager k8sManager;
private readonly List<OnlineCodexNode> activeMetricsNodes = new List<OnlineCodexNode>();
private PrometheusInfo? activePrometheus;
public MetricsAggregator(TestLog log, K8sManager k8sManager)
@ -15,71 +15,57 @@ namespace CodexDistTestCore
this.k8sManager = k8sManager;
}
public MetricsAccess BeginCollectingMetricsFor(IOnlineCodexNode[] nodes)
public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
{
EnsurePrometheusPod();
if (activePrometheus != null)
{
Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :(");
throw new InvalidOperationException();
}
AddNewCodexNodes(nodes);
log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
// Get IPS and ports from all nodes, format prometheus configuration
var config = GeneratePrometheusConfig();
// Create config file inside prometheus pod
k8sManager.UploadFileToPod(
activePrometheus!.PodInfo.Name,
K8sPrometheusSpecs.ContainerName,
config,
K8sPrometheusSpecs.ConfigFilepath);
var config = GeneratePrometheusConfig(nodes);
StartPrometheusPod(config);
// HTTP POST request to the /-/reload endpoint (when the --web.enable-lifecycle flag is enabled).
return new MetricsAccess();
log.Log("Metrics service started.");
return new MetricsAccess(activePrometheus!);
}
public void DownloadAllMetrics()
{
}
private void EnsurePrometheusPod()
private void StartPrometheusPod(string config)
{
if (activePrometheus != null) return;
activePrometheus = k8sManager.BringOnlinePrometheus();
activePrometheus = k8sManager.BringOnlinePrometheus(config);
}
private void AddNewCodexNodes(IOnlineCodexNode[] nodes)
private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
{
activeMetricsNodes.AddRange(nodes.Where(n => !activeMetricsNodes.Contains(n)).Cast<OnlineCodexNode>());
}
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
config += " - 'prometheus:9090'\n";
private Stream GeneratePrometheusConfig()
{
var stream = new MemoryStream();
using var writer = new StreamWriter(stream);
writer.WriteLine("global:");
writer.WriteLine(" scrape_interval: 30s");
writer.WriteLine(" scrape_timeout: 10s");
writer.WriteLine("");
writer.WriteLine("rule_files:");
writer.WriteLine(" - alert.yml");
writer.WriteLine("");
writer.WriteLine("scrape_configs:");
writer.WriteLine(" - job_name: services");
writer.WriteLine(" metrics_path: /metrics");
writer.WriteLine(" static_configs:");
writer.WriteLine(" - targets:");
writer.WriteLine(" - 'prometheus:9090'");
foreach (var node in activeMetricsNodes)
foreach (var node in nodes)
{
var ip = node.Group.PodInfo!.Ip;
var port = node.Container.ServicePort;
writer.WriteLine($" - '{ip}:{port}'");
var port = node.Container.MetricsPort;
config += $" - '{ip}:{port}'\n";
}
return stream;
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
}
public class PrometheusInfo

2
Tests/BasicTests/SimpleTests.cs
View File

@ -43,8 +43,6 @@ namespace Tests.BasicTests
var secondary = group[1];
primary.ConnectToPeer(secondary);
Thread.Sleep(10000);
AssertWithTimeout(
() => metrics.GetMostRecentInt("libp2p_peers", primary),
isEqualTo: 1,