status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Restores pod labels in deployer. Adds metrics assert for twoclient continuous test.

This commit is contained in:
benbierens 2023-09-26 14:32:28 +02:00
parent 52345adb49
commit 0a2bca9f52
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
10 changed files with 159 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs
View File

@ -2,7 +2,6 @@
using GethPlugin;
using KubernetesWorkflow;
using Logging;
using Nethereum.Contracts;
using Utils;
namespace CodexContractsPlugin
@ -54,7 +53,7 @@ namespace CodexContractsPlugin
WaitUntil(() =>
{
var logHandler = new ContractsReadyLogHandler(tools.GetLog());
workflow.DownloadContainerLog(container, logHandler, null);
workflow.DownloadContainerLog(container, logHandler, 100);
return logHandler.Found;
});
Log("Contracts deployed. Extracting addresses...");

2
ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs
View File

@ -109,8 +109,6 @@ namespace CodexPlugin
{
AddEnvVar("CODEX_NODENAME", config.NameOverride);
}
AddPodLabel("codexid", Image);
}
private ByteSize GetVolumeCapacity(CodexStartupConfig config)

5
ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs
View File

@ -16,6 +16,11 @@ namespace MetricsPlugin
return Plugin(ci).DeployMetricsCollector(scrapeTargets);
}
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
{
return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
}
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IMetricsScrapeTarget scrapeTarget)
{
return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);

51
ProjectPlugins/MetricsPlugin/MetricsQuery.cs
View File

@ -1,5 +1,6 @@
using Core;
using KubernetesWorkflow;
using Logging;
using System.Globalization;
namespace MetricsPlugin
@ -7,11 +8,13 @@ namespace MetricsPlugin
public class MetricsQuery
{
private readonly IHttp http;
private readonly ILog log;
public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
{
RunningContainer = runningContainer;
http = tools.CreateHttp(RunningContainer.Address, "api/v1");
log = tools.GetLog();
}
public RunningContainer RunningContainer { get; }
@ -21,7 +24,7 @@ namespace MetricsPlugin
var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
if (response == null) return null;
return new Metrics
var result = new Metrics
{
Sets = response.data.result.Select(r =>
{
@ -32,20 +35,27 @@ namespace MetricsPlugin
};
}).ToArray()
};
Log(target, metricName, result);
return result;
}
public Metrics? GetMetrics(string metricName)
{
var response = GetAll(metricName);
if (response == null) return null;
return MapResponseToMetrics(response);
var result = MapResponseToMetrics(response);
Log(metricName, result);
return result;
}
public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return MapResponseToMetrics(response);
var result = MapResponseToMetrics(response);
Log(target, result);
return result;
}
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
@ -135,11 +145,36 @@ namespace MetricsPlugin
var unixSeconds = ToValue(v);
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
private void Log(IMetricsScrapeTarget target, string metricName, Metrics result)
{
Log($"{target.Name} '{metricName}' = {result}");
}
private void Log(string metricName, Metrics result)
{
Log($"'{metricName}' = {result}");
}
private void Log(IMetricsScrapeTarget target, Metrics result)
{
Log($"{target.Name} => {result}");
}
private void Log(string msg)
{
log.Log(msg);
}
}
public class Metrics
{
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
public override string ToString()
{
return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
}
}
public class MetricsSet
@ -147,12 +182,22 @@ namespace MetricsPlugin
public string Name { get; set; } = string.Empty;
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
public override string ToString()
{
return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
}
}
public class MetricsSetValue
{
public DateTime Timestamp { get; set; }
public double Value { get; set; }
public override string ToString()
{
return $"<{Timestamp.ToString("o")}={Value}>";
}
}
public class PrometheusQueryResponse

15
Tests/CodexContinuousTests/ContinuousTest.cs
View File

@ -3,6 +3,7 @@ using Core;
using DistTestCore;
using FileUtils;
using Logging;
using MetricsPlugin;
namespace ContinuousTests
{
@ -47,6 +48,20 @@ namespace ContinuousTests
public CancellationToken CancelToken { get; private set; } = new CancellationToken();
public NodeRunner NodeRunner { get; private set; } = null!;
public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target)
{
return CreateMetricsAccess(target.MetricsScrapeTarget);
}
public IMetricsAccess CreateMetricsAccess(IMetricsScrapeTarget target)
{
if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment.");
var entryPointFactory = new EntryPointFactory();
var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.CodexDeployment.Metadata.KubeNamespace, Log);
return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.CodexDeployment.PrometheusContainer, target);
}
public abstract int RequiredNumberOfNodes { get; }
public abstract TimeSpan RunTestEvery { get; }
public abstract TestFailMode TestFailMode { get; }

29
Tests/CodexContinuousTests/Tests/TwoClientTest.cs
View File

@ -7,6 +7,8 @@ namespace ContinuousTests.Tests
{
public class TwoClientTest : ContinuousTest
{
private const string BytesStoredMetric = "codexRepostoreBytesUsed";
public override int RequiredNumberOfNodes => 2;
public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
@ -17,10 +19,14 @@ namespace ContinuousTests.Tests
[TestMoment(t: Zero)]
public void UploadTestFile()
{
file = FileManager.GenerateFile(80.MB());
var size = 80.MB();
file = FileManager.GenerateFile(size);
AssertBytesStoredMetric(size, Nodes[0], () =>
{
cid = Nodes[0].UploadFile(file);
Assert.That(cid, Is.Not.Null);
});
}
[TestMoment(t: 10)]
@ -30,5 +36,26 @@ namespace ContinuousTests.Tests
file.AssertIsEqual(dl);
}
private void AssertBytesStoredMetric(ByteSize uploadedSize, ICodexNode node, Action action)
{
var lowExpected = uploadedSize.SizeInBytes;
var highExpected = uploadedSize.SizeInBytes * 1.2;
var metrics = CreateMetricsAccess(node);
var before = metrics.GetMetric(BytesStoredMetric);
action();
Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}.");
Time.WaitUntil(() =>
{
var after = metrics.GetMetric(BytesStoredMetric);
var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value);
return highExpected > newBytes && newBytes > lowExpected;
});
}
}
}

56
Tests/CodexTests/BasicTests/ContinuousSubstitute.cs
View File

@ -1,5 +1,8 @@
using CodexPlugin;
using DistTestCore;
using CodexContractsPlugin;
using CodexPlugin;
using GethPlugin;
using KubernetesWorkflow;
using MetricsPlugin;
using NUnit.Framework;
using Utils;
@ -12,24 +15,28 @@ namespace Tests.BasicTests
[Test]
public void ContinuousTestSubstitute()
{
var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth"));
var contract = Ci.StartCodexContracts(geth);
var group = AddCodex(5, o => o
//.EnableMetrics()
//.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceNumber(10000)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.EnableMetrics()
.EnableMarketplace(geth, contract, 10.Eth(), 100000.TestTokens(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(5))
.WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10))
.WithBlockMaintenanceNumber(100)
.WithStorageQuota(1.GB()));
var nodes = group.Cast<CodexNode>().ToArray();
var rc = Ci.DeployMetricsCollector(nodes);
foreach (var node in nodes)
{
//node.Marketplace.MakeStorageAvailable(
//size: 500.MB(),
//minPricePerBytePerSecond: 1.TestTokens(),
//maxCollateral: 1024.TestTokens(),
//maxDuration: TimeSpan.FromMinutes(5));
node.Marketplace.MakeStorageAvailable(
size: 500.MB(),
minPriceForTotalSpace: 500.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
@ -40,7 +47,7 @@ namespace Tests.BasicTests
var secondary = allNodes.PickOneRandom();
Log("Run Test");
PerformTest(primary, secondary);
PerformTest(primary, secondary, rc);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
@ -104,14 +111,31 @@ namespace Tests.BasicTests
private ByteSize fileSize = 80.MB();
private void PerformTest(ICodexNode primary, ICodexNode secondary)
private const string BytesStoredMetric = "codexRepostoreBytesUsed";
private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainer rc)
{
ScopedTestFiles(() =>
{
var testFile = GenerateTestFile(fileSize);
var metrics = Ci.WrapMetricsCollector(rc, primary);
var beforeBytesStored = metrics.GetMetric(BytesStoredMetric);
var contentId = primary.UploadFile(testFile);
var low = fileSize.SizeInBytes;
var high = low * 1.2;
Log("looking for: " + low + " < " + high);
Time.WaitUntil(() =>
{
var afterBytesStored = metrics.GetMetric(BytesStoredMetric);
var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
return high > newBytes && newBytes > low;
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
@ -123,7 +147,7 @@ namespace Tests.BasicTests
{
var blockExpirationTime = TimeSpan.FromMinutes(3);
var group = AddCodex(3, o => o
//.EnableMetrics()
.EnableMetrics()
.WithBlockTTL(blockExpirationTime)
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceNumber(10000)

7
Tests/DistTestCore/TestLifecycle.cs
View File

@ -11,6 +11,7 @@ namespace DistTestCore
private const string TestsType = "dist-tests";
private readonly DateTime testStart;
private readonly EntryPoint entryPoint;
private readonly Dictionary<string, string> metadata;
private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
@ -21,6 +22,7 @@ namespace DistTestCore
testStart = DateTime.UtcNow;
entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
metadata = entryPoint.GetPluginMetadata();
CoreInterface = entryPoint.CreateInterface();
log.WriteLogTag();
@ -78,6 +80,11 @@ namespace DistTestCore
recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());
recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName());
foreach (var pair in metadata)
{
recipe.PodLabels.Add(pair.Key, pair.Value);
}
}
public void DownloadAllLogs()

5
Tools/CodexNetDeployer/Deployer.cs
View File

@ -85,9 +85,10 @@ namespace CodexNetDeployer
retryDelay: TimeSpan.FromSeconds(3),
kubernetesNamespace: config.KubeNamespace);
configuration.Hooks = new K8sHook(config.TestsTypePodLabel);
var result = new EntryPoint(log, configuration, string.Empty);
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata());
return new EntryPoint(log, configuration, string.Empty);
return result;
}
private RunningContainer? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)

9
Tools/CodexNetDeployer/K8sHook.cs
View File

@ -6,10 +6,12 @@ namespace CodexNetDeployer
public class K8sHook : IK8sHooks
{
private readonly string testsTypeLabel;
private readonly Dictionary<string, string> metadata;
public K8sHook(string testsTypeLabel)
public K8sHook(string testsTypeLabel, Dictionary<string, string> metadata)
{
this.testsTypeLabel = testsTypeLabel;
this.metadata = metadata;
}
public void OnContainersStarted(RunningContainers rc)
@ -25,6 +27,11 @@ namespace CodexNetDeployer
recipe.PodLabels.Add("tests-type", testsTypeLabel);
recipe.PodLabels.Add("runid", NameUtils.GetRunId());
recipe.PodLabels.Add("testid", NameUtils.GetTestId());
foreach (var pair in metadata)
{
recipe.PodLabels.Add(pair.Key, pair.Value);
}
}
}
}