Restores pod labels in deployer. Adds metrics assert for twoclient continuous test.
This commit is contained in:
parent
52345adb49
commit
0a2bca9f52
|
@ -2,7 +2,6 @@
|
|||
using GethPlugin;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using Nethereum.Contracts;
|
||||
using Utils;
|
||||
|
||||
namespace CodexContractsPlugin
|
||||
|
@ -54,7 +53,7 @@ namespace CodexContractsPlugin
|
|||
WaitUntil(() =>
|
||||
{
|
||||
var logHandler = new ContractsReadyLogHandler(tools.GetLog());
|
||||
workflow.DownloadContainerLog(container, logHandler, null);
|
||||
workflow.DownloadContainerLog(container, logHandler, 100);
|
||||
return logHandler.Found;
|
||||
});
|
||||
Log("Contracts deployed. Extracting addresses...");
|
||||
|
|
|
@ -109,8 +109,6 @@ namespace CodexPlugin
|
|||
{
|
||||
AddEnvVar("CODEX_NODENAME", config.NameOverride);
|
||||
}
|
||||
|
||||
AddPodLabel("codexid", Image);
|
||||
}
|
||||
|
||||
private ByteSize GetVolumeCapacity(CodexStartupConfig config)
|
||||
|
|
|
@ -16,6 +16,11 @@ namespace MetricsPlugin
|
|||
return Plugin(ci).DeployMetricsCollector(scrapeTargets);
|
||||
}
|
||||
|
||||
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
|
||||
{
|
||||
return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
|
||||
}
|
||||
|
||||
public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IMetricsScrapeTarget scrapeTarget)
|
||||
{
|
||||
return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
using Core;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
using System.Globalization;
|
||||
|
||||
namespace MetricsPlugin
|
||||
|
@ -7,11 +8,13 @@ namespace MetricsPlugin
|
|||
public class MetricsQuery
|
||||
{
|
||||
private readonly IHttp http;
|
||||
private readonly ILog log;
|
||||
|
||||
public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
|
||||
{
|
||||
RunningContainer = runningContainer;
|
||||
http = tools.CreateHttp(RunningContainer.Address, "api/v1");
|
||||
log = tools.GetLog();
|
||||
}
|
||||
|
||||
public RunningContainer RunningContainer { get; }
|
||||
|
@ -21,7 +24,7 @@ namespace MetricsPlugin
|
|||
var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
|
||||
if (response == null) return null;
|
||||
|
||||
return new Metrics
|
||||
var result = new Metrics
|
||||
{
|
||||
Sets = response.data.result.Select(r =>
|
||||
{
|
||||
|
@ -32,20 +35,27 @@ namespace MetricsPlugin
|
|||
};
|
||||
}).ToArray()
|
||||
};
|
||||
|
||||
Log(target, metricName, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
public Metrics? GetMetrics(string metricName)
|
||||
{
|
||||
var response = GetAll(metricName);
|
||||
if (response == null) return null;
|
||||
return MapResponseToMetrics(response);
|
||||
var result = MapResponseToMetrics(response);
|
||||
Log(metricName, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
|
||||
{
|
||||
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
|
||||
if (response.status != "success") return null;
|
||||
return MapResponseToMetrics(response);
|
||||
var result = MapResponseToMetrics(response);
|
||||
Log(target, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
|
||||
|
@ -135,11 +145,36 @@ namespace MetricsPlugin
|
|||
var unixSeconds = ToValue(v);
|
||||
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
|
||||
}
|
||||
|
||||
private void Log(IMetricsScrapeTarget target, string metricName, Metrics result)
|
||||
{
|
||||
Log($"{target.Name} '{metricName}' = {result}");
|
||||
}
|
||||
|
||||
private void Log(string metricName, Metrics result)
|
||||
{
|
||||
Log($"'{metricName}' = {result}");
|
||||
}
|
||||
|
||||
private void Log(IMetricsScrapeTarget target, Metrics result)
|
||||
{
|
||||
Log($"{target.Name} => {result}");
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
log.Log(msg);
|
||||
}
|
||||
}
|
||||
|
||||
public class Metrics
|
||||
{
|
||||
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsSet
|
||||
|
@ -147,12 +182,22 @@ namespace MetricsPlugin
|
|||
public string Name { get; set; } = string.Empty;
|
||||
public string Instance { get; set; } = string.Empty;
|
||||
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsSetValue
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
public double Value { get; set; }
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"<{Timestamp.ToString("o")}={Value}>";
|
||||
}
|
||||
}
|
||||
|
||||
public class PrometheusQueryResponse
|
||||
|
|
|
@ -3,6 +3,7 @@ using Core;
|
|||
using DistTestCore;
|
||||
using FileUtils;
|
||||
using Logging;
|
||||
using MetricsPlugin;
|
||||
|
||||
namespace ContinuousTests
|
||||
{
|
||||
|
@ -47,6 +48,20 @@ namespace ContinuousTests
|
|||
public CancellationToken CancelToken { get; private set; } = new CancellationToken();
|
||||
public NodeRunner NodeRunner { get; private set; } = null!;
|
||||
|
||||
public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target)
|
||||
{
|
||||
return CreateMetricsAccess(target.MetricsScrapeTarget);
|
||||
}
|
||||
|
||||
public IMetricsAccess CreateMetricsAccess(IMetricsScrapeTarget target)
|
||||
{
|
||||
if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment.");
|
||||
|
||||
var entryPointFactory = new EntryPointFactory();
|
||||
var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.CodexDeployment.Metadata.KubeNamespace, Log);
|
||||
return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.CodexDeployment.PrometheusContainer, target);
|
||||
}
|
||||
|
||||
public abstract int RequiredNumberOfNodes { get; }
|
||||
public abstract TimeSpan RunTestEvery { get; }
|
||||
public abstract TestFailMode TestFailMode { get; }
|
||||
|
|
|
@ -7,6 +7,8 @@ namespace ContinuousTests.Tests
|
|||
{
|
||||
public class TwoClientTest : ContinuousTest
|
||||
{
|
||||
private const string BytesStoredMetric = "codexRepostoreBytesUsed";
|
||||
|
||||
public override int RequiredNumberOfNodes => 2;
|
||||
public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
|
||||
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
|
||||
|
@ -17,10 +19,14 @@ namespace ContinuousTests.Tests
|
|||
[TestMoment(t: Zero)]
|
||||
public void UploadTestFile()
|
||||
{
|
||||
file = FileManager.GenerateFile(80.MB());
|
||||
var size = 80.MB();
|
||||
file = FileManager.GenerateFile(size);
|
||||
|
||||
cid = Nodes[0].UploadFile(file);
|
||||
Assert.That(cid, Is.Not.Null);
|
||||
AssertBytesStoredMetric(size, Nodes[0], () =>
|
||||
{
|
||||
cid = Nodes[0].UploadFile(file);
|
||||
Assert.That(cid, Is.Not.Null);
|
||||
});
|
||||
}
|
||||
|
||||
[TestMoment(t: 10)]
|
||||
|
@ -30,5 +36,26 @@ namespace ContinuousTests.Tests
|
|||
|
||||
file.AssertIsEqual(dl);
|
||||
}
|
||||
|
||||
private void AssertBytesStoredMetric(ByteSize uploadedSize, ICodexNode node, Action action)
|
||||
{
|
||||
var lowExpected = uploadedSize.SizeInBytes;
|
||||
var highExpected = uploadedSize.SizeInBytes * 1.2;
|
||||
|
||||
var metrics = CreateMetricsAccess(node);
|
||||
var before = metrics.GetMetric(BytesStoredMetric);
|
||||
|
||||
action();
|
||||
|
||||
Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}.");
|
||||
|
||||
Time.WaitUntil(() =>
|
||||
{
|
||||
var after = metrics.GetMetric(BytesStoredMetric);
|
||||
var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value);
|
||||
|
||||
return highExpected > newBytes && newBytes > lowExpected;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,8 @@
|
|||
using CodexPlugin;
|
||||
using DistTestCore;
|
||||
using CodexContractsPlugin;
|
||||
using CodexPlugin;
|
||||
using GethPlugin;
|
||||
using KubernetesWorkflow;
|
||||
using MetricsPlugin;
|
||||
using NUnit.Framework;
|
||||
using Utils;
|
||||
|
||||
|
@ -12,24 +15,28 @@ namespace Tests.BasicTests
|
|||
[Test]
|
||||
public void ContinuousTestSubstitute()
|
||||
{
|
||||
var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth"));
|
||||
var contract = Ci.StartCodexContracts(geth);
|
||||
|
||||
var group = AddCodex(5, o => o
|
||||
//.EnableMetrics()
|
||||
//.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
|
||||
.WithBlockTTL(TimeSpan.FromMinutes(2))
|
||||
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
|
||||
.WithBlockMaintenanceNumber(10000)
|
||||
.WithBlockTTL(TimeSpan.FromMinutes(2))
|
||||
.EnableMetrics()
|
||||
.EnableMarketplace(geth, contract, 10.Eth(), 100000.TestTokens(), isValidator: true)
|
||||
.WithBlockTTL(TimeSpan.FromMinutes(5))
|
||||
.WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10))
|
||||
.WithBlockMaintenanceNumber(100)
|
||||
.WithStorageQuota(1.GB()));
|
||||
|
||||
var nodes = group.Cast<CodexNode>().ToArray();
|
||||
|
||||
var rc = Ci.DeployMetricsCollector(nodes);
|
||||
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
//node.Marketplace.MakeStorageAvailable(
|
||||
//size: 500.MB(),
|
||||
//minPricePerBytePerSecond: 1.TestTokens(),
|
||||
//maxCollateral: 1024.TestTokens(),
|
||||
//maxDuration: TimeSpan.FromMinutes(5));
|
||||
node.Marketplace.MakeStorageAvailable(
|
||||
size: 500.MB(),
|
||||
minPriceForTotalSpace: 500.TestTokens(),
|
||||
maxCollateral: 1024.TestTokens(),
|
||||
maxDuration: TimeSpan.FromMinutes(5));
|
||||
}
|
||||
|
||||
var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
|
||||
|
@ -40,7 +47,7 @@ namespace Tests.BasicTests
|
|||
var secondary = allNodes.PickOneRandom();
|
||||
|
||||
Log("Run Test");
|
||||
PerformTest(primary, secondary);
|
||||
PerformTest(primary, secondary, rc);
|
||||
|
||||
Thread.Sleep(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
@ -81,7 +88,7 @@ namespace Tests.BasicTests
|
|||
var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
|
||||
|
||||
var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
|
||||
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
Assert.Fail(string.Join(Environment.NewLine, errors));
|
||||
|
@ -104,26 +111,43 @@ namespace Tests.BasicTests
|
|||
|
||||
private ByteSize fileSize = 80.MB();
|
||||
|
||||
private void PerformTest(ICodexNode primary, ICodexNode secondary)
|
||||
private const string BytesStoredMetric = "codexRepostoreBytesUsed";
|
||||
|
||||
private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainer rc)
|
||||
{
|
||||
ScopedTestFiles(() =>
|
||||
{
|
||||
var testFile = GenerateTestFile(fileSize);
|
||||
|
||||
var metrics = Ci.WrapMetricsCollector(rc, primary);
|
||||
var beforeBytesStored = metrics.GetMetric(BytesStoredMetric);
|
||||
|
||||
var contentId = primary.UploadFile(testFile);
|
||||
|
||||
var low = fileSize.SizeInBytes;
|
||||
var high = low * 1.2;
|
||||
Log("looking for: " + low + " < " + high);
|
||||
|
||||
Time.WaitUntil(() =>
|
||||
{
|
||||
var afterBytesStored = metrics.GetMetric(BytesStoredMetric);
|
||||
var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
|
||||
|
||||
return high > newBytes && newBytes > low;
|
||||
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
|
||||
|
||||
var downloadedFile = secondary.DownloadContent(contentId);
|
||||
|
||||
testFile.AssertIsEqual(downloadedFile);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void HoldMyBeerTest()
|
||||
{
|
||||
var blockExpirationTime = TimeSpan.FromMinutes(3);
|
||||
var group = AddCodex(3, o => o
|
||||
//.EnableMetrics()
|
||||
.EnableMetrics()
|
||||
.WithBlockTTL(blockExpirationTime)
|
||||
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
|
||||
.WithBlockMaintenanceNumber(10000)
|
||||
|
|
|
@ -11,6 +11,7 @@ namespace DistTestCore
|
|||
private const string TestsType = "dist-tests";
|
||||
private readonly DateTime testStart;
|
||||
private readonly EntryPoint entryPoint;
|
||||
private readonly Dictionary<string, string> metadata;
|
||||
private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
|
||||
|
||||
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
|
||||
|
@ -21,6 +22,7 @@ namespace DistTestCore
|
|||
testStart = DateTime.UtcNow;
|
||||
|
||||
entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
|
||||
metadata = entryPoint.GetPluginMetadata();
|
||||
CoreInterface = entryPoint.CreateInterface();
|
||||
|
||||
log.WriteLogTag();
|
||||
|
@ -78,6 +80,11 @@ namespace DistTestCore
|
|||
recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
|
||||
recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());
|
||||
recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName());
|
||||
|
||||
foreach (var pair in metadata)
|
||||
{
|
||||
recipe.PodLabels.Add(pair.Key, pair.Value);
|
||||
}
|
||||
}
|
||||
|
||||
public void DownloadAllLogs()
|
||||
|
|
|
@ -85,9 +85,10 @@ namespace CodexNetDeployer
|
|||
retryDelay: TimeSpan.FromSeconds(3),
|
||||
kubernetesNamespace: config.KubeNamespace);
|
||||
|
||||
configuration.Hooks = new K8sHook(config.TestsTypePodLabel);
|
||||
var result = new EntryPoint(log, configuration, string.Empty);
|
||||
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata());
|
||||
|
||||
return new EntryPoint(log, configuration, string.Empty);
|
||||
return result;
|
||||
}
|
||||
|
||||
private RunningContainer? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
|
||||
|
|
|
@ -6,10 +6,12 @@ namespace CodexNetDeployer
|
|||
public class K8sHook : IK8sHooks
|
||||
{
|
||||
private readonly string testsTypeLabel;
|
||||
private readonly Dictionary<string, string> metadata;
|
||||
|
||||
public K8sHook(string testsTypeLabel)
|
||||
public K8sHook(string testsTypeLabel, Dictionary<string, string> metadata)
|
||||
{
|
||||
this.testsTypeLabel = testsTypeLabel;
|
||||
this.metadata = metadata;
|
||||
}
|
||||
|
||||
public void OnContainersStarted(RunningContainers rc)
|
||||
|
@ -25,6 +27,11 @@ namespace CodexNetDeployer
|
|||
recipe.PodLabels.Add("tests-type", testsTypeLabel);
|
||||
recipe.PodLabels.Add("runid", NameUtils.GetRunId());
|
||||
recipe.PodLabels.Add("testid", NameUtils.GetTestId());
|
||||
|
||||
foreach (var pair in metadata)
|
||||
{
|
||||
recipe.PodLabels.Add(pair.Key, pair.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue