diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index f6fae82c..ea8fe4b7 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -2,7 +2,6 @@ using GethPlugin; using KubernetesWorkflow; using Logging; -using Nethereum.Contracts; using Utils; namespace CodexContractsPlugin @@ -54,7 +53,7 @@ namespace CodexContractsPlugin WaitUntil(() => { var logHandler = new ContractsReadyLogHandler(tools.GetLog()); - workflow.DownloadContainerLog(container, logHandler, null); + workflow.DownloadContainerLog(container, logHandler, 100); return logHandler.Found; }); Log("Contracts deployed. Extracting addresses..."); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 3f230ba3..bee2d4a8 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -109,8 +109,6 @@ namespace CodexPlugin { AddEnvVar("CODEX_NODENAME", config.NameOverride); } - - AddPodLabel("codexid", Image); } private ByteSize GetVolumeCapacity(CodexStartupConfig config) diff --git a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs index 0c320075..7db1f3f5 100644 --- a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs @@ -16,6 +16,11 @@ namespace MetricsPlugin return Plugin(ci).DeployMetricsCollector(scrapeTargets); } + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IHasMetricsScrapeTarget scrapeTarget) + { + return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget); + } + public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IMetricsScrapeTarget scrapeTarget) { return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget); diff --git a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs index 7d634776..fbbb827d 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs @@ -1,5 +1,6 @@ using Core; using KubernetesWorkflow; +using Logging; using System.Globalization; namespace MetricsPlugin @@ -7,11 +8,13 @@ namespace MetricsPlugin public class MetricsQuery { private readonly IHttp http; + private readonly ILog log; public MetricsQuery(IPluginTools tools, RunningContainer runningContainer) { RunningContainer = runningContainer; http = tools.CreateHttp(RunningContainer.Address, "api/v1"); + log = tools.GetLog(); } public RunningContainer RunningContainer { get; } @@ -21,7 +24,7 @@ namespace MetricsPlugin var response = GetLastOverTime(metricName, GetInstanceStringForNode(target)); if (response == null) return null; - return new Metrics + var result = new Metrics { Sets = response.data.result.Select(r => { @@ -32,20 +35,27 @@ namespace MetricsPlugin }; }).ToArray() }; + + Log(target, metricName, result); + return result; } public Metrics? GetMetrics(string metricName) { var response = GetAll(metricName); if (response == null) return null; - return MapResponseToMetrics(response); + var result = MapResponseToMetrics(response); + Log(metricName, result); + return result; } public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target) { var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}"); if (response.status != "success") return null; - return MapResponseToMetrics(response); + var result = MapResponseToMetrics(response); + Log(target, result); + return result; } private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) @@ -135,11 +145,36 @@ namespace MetricsPlugin var unixSeconds = ToValue(v); return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); } + + private void Log(IMetricsScrapeTarget target, string metricName, Metrics result) + { + Log($"{target.Name} '{metricName}' = {result}"); + } + + private void Log(string metricName, Metrics result) + { + Log($"'{metricName}' = {result}"); + } + + private void Log(IMetricsScrapeTarget target, Metrics result) + { + Log($"{target.Name} => {result}"); + } + + private void Log(string msg) + { + log.Log(msg); + } } public class Metrics { public MetricsSet[] Sets { get; set; } = Array.Empty(); + + public override string ToString() + { + return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]"; + } } public class MetricsSet @@ -147,12 +182,22 @@ namespace MetricsPlugin public string Name { get; set; } = string.Empty; public string Instance { get; set; } = string.Empty; public MetricsSetValue[] Values { get; set; } = Array.Empty(); + + public override string ToString() + { + return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}"; + } } public class MetricsSetValue { public DateTime Timestamp { get; set; } public double Value { get; set; } + + public override string ToString() + { + return $"<{Timestamp.ToString("o")}={Value}>"; + } } public class PrometheusQueryResponse diff --git a/Tests/CodexContinuousTests/ContinuousTest.cs b/Tests/CodexContinuousTests/ContinuousTest.cs index 2bef6dba..5b746682 100644 --- a/Tests/CodexContinuousTests/ContinuousTest.cs +++ b/Tests/CodexContinuousTests/ContinuousTest.cs @@ -3,6 +3,7 @@ using Core; using DistTestCore; using FileUtils; using Logging; +using MetricsPlugin; namespace ContinuousTests { @@ -47,6 +48,20 @@ namespace ContinuousTests public CancellationToken CancelToken { get; private set; } = new CancellationToken(); public NodeRunner NodeRunner { get; private set; } = null!; + public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target) + { + return CreateMetricsAccess(target.MetricsScrapeTarget); + } + + public IMetricsAccess CreateMetricsAccess(IMetricsScrapeTarget target) + { + if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment."); + + var entryPointFactory = new EntryPointFactory(); + var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.CodexDeployment.Metadata.KubeNamespace, Log); + return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.CodexDeployment.PrometheusContainer, target); + } + public abstract int RequiredNumberOfNodes { get; } public abstract TimeSpan RunTestEvery { get; } public abstract TestFailMode TestFailMode { get; } diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index 53cff4b6..12a4d3fd 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -7,6 +7,8 @@ namespace ContinuousTests.Tests { public class TwoClientTest : ContinuousTest { + private const string BytesStoredMetric = "codexRepostoreBytesUsed"; + public override int RequiredNumberOfNodes => 2; public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; @@ -17,10 +19,14 @@ namespace ContinuousTests.Tests [TestMoment(t: Zero)] public void UploadTestFile() { - file = FileManager.GenerateFile(80.MB()); + var size = 80.MB(); + file = FileManager.GenerateFile(size); - cid = Nodes[0].UploadFile(file); - Assert.That(cid, Is.Not.Null); + AssertBytesStoredMetric(size, Nodes[0], () => + { + cid = Nodes[0].UploadFile(file); + Assert.That(cid, Is.Not.Null); + }); } [TestMoment(t: 10)] @@ -30,5 +36,26 @@ namespace ContinuousTests.Tests file.AssertIsEqual(dl); } + + private void AssertBytesStoredMetric(ByteSize uploadedSize, ICodexNode node, Action action) + { + var lowExpected = uploadedSize.SizeInBytes; + var highExpected = uploadedSize.SizeInBytes * 1.2; + + var metrics = CreateMetricsAccess(node); + var before = metrics.GetMetric(BytesStoredMetric); + + action(); + + Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}."); + + Time.WaitUntil(() => + { + var after = metrics.GetMetric(BytesStoredMetric); + var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value); + + return highExpected > newBytes && newBytes > lowExpected; + }); + } } } diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 5a4b32d0..1258f55a 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -1,5 +1,8 @@ -using CodexPlugin; -using DistTestCore; +using CodexContractsPlugin; +using CodexPlugin; +using GethPlugin; +using KubernetesWorkflow; +using MetricsPlugin; using NUnit.Framework; using Utils; @@ -12,24 +15,28 @@ namespace Tests.BasicTests [Test] public void ContinuousTestSubstitute() { + var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth")); + var contract = Ci.StartCodexContracts(geth); + var group = AddCodex(5, o => o - //.EnableMetrics() - //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) - .WithBlockTTL(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceNumber(10000) - .WithBlockTTL(TimeSpan.FromMinutes(2)) + .EnableMetrics() + .EnableMarketplace(geth, contract, 10.Eth(), 100000.TestTokens(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(5)) + .WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10)) + .WithBlockMaintenanceNumber(100) .WithStorageQuota(1.GB())); var nodes = group.Cast().ToArray(); + var rc = Ci.DeployMetricsCollector(nodes); + foreach (var node in nodes) { - //node.Marketplace.MakeStorageAvailable( - //size: 500.MB(), - //minPricePerBytePerSecond: 1.TestTokens(), - //maxCollateral: 1024.TestTokens(), - //maxDuration: TimeSpan.FromMinutes(5)); + node.Marketplace.MakeStorageAvailable( + size: 500.MB(), + minPriceForTotalSpace: 500.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); } var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); @@ -40,7 +47,7 @@ namespace Tests.BasicTests var secondary = allNodes.PickOneRandom(); Log("Run Test"); - PerformTest(primary, secondary); + PerformTest(primary, secondary, rc); Thread.Sleep(TimeSpan.FromSeconds(5)); } @@ -81,7 +88,7 @@ namespace Tests.BasicTests var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray(); var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray(); - + if (errors.Any()) { Assert.Fail(string.Join(Environment.NewLine, errors)); @@ -104,26 +111,43 @@ namespace Tests.BasicTests private ByteSize fileSize = 80.MB(); - private void PerformTest(ICodexNode primary, ICodexNode secondary) + private const string BytesStoredMetric = "codexRepostoreBytesUsed"; + + private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainer rc) { ScopedTestFiles(() => { var testFile = GenerateTestFile(fileSize); + var metrics = Ci.WrapMetricsCollector(rc, primary); + var beforeBytesStored = metrics.GetMetric(BytesStoredMetric); + var contentId = primary.UploadFile(testFile); + var low = fileSize.SizeInBytes; + var high = low * 1.2; + Log("looking for: " + low + " < " + high); + + Time.WaitUntil(() => + { + var afterBytesStored = metrics.GetMetric(BytesStoredMetric); + var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value); + + return high > newBytes && newBytes > low; + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2)); + var downloadedFile = secondary.DownloadContent(contentId); testFile.AssertIsEqual(downloadedFile); }); } - + [Test] public void HoldMyBeerTest() { var blockExpirationTime = TimeSpan.FromMinutes(3); var group = AddCodex(3, o => o - //.EnableMetrics() + .EnableMetrics() .WithBlockTTL(blockExpirationTime) .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) .WithBlockMaintenanceNumber(10000) diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index c0904c11..47a17b38 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -11,6 +11,7 @@ namespace DistTestCore private const string TestsType = "dist-tests"; private readonly DateTime testStart; private readonly EntryPoint entryPoint; + private readonly Dictionary metadata; private readonly List runningContainers = new List(); public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace) @@ -21,6 +22,7 @@ namespace DistTestCore testStart = DateTime.UtcNow; entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet); + metadata = entryPoint.GetPluginMetadata(); CoreInterface = entryPoint.CreateInterface(); log.WriteLogTag(); @@ -78,6 +80,11 @@ namespace DistTestCore recipe.PodLabels.Add("category", NameUtils.GetCategoryName()); recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName()); recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName()); + + foreach (var pair in metadata) + { + recipe.PodLabels.Add(pair.Key, pair.Value); + } } public void DownloadAllLogs() diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 958df708..5c1365f0 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -85,9 +85,10 @@ namespace CodexNetDeployer retryDelay: TimeSpan.FromSeconds(3), kubernetesNamespace: config.KubeNamespace); - configuration.Hooks = new K8sHook(config.TestsTypePodLabel); + var result = new EntryPoint(log, configuration, string.Empty); + configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata()); - return new EntryPoint(log, configuration, string.Empty); + return result; } private RunningContainer? StartMetricsService(CoreInterface ci, List startResults) diff --git a/Tools/CodexNetDeployer/K8sHook.cs b/Tools/CodexNetDeployer/K8sHook.cs index b81078fc..f89a3fc3 100644 --- a/Tools/CodexNetDeployer/K8sHook.cs +++ b/Tools/CodexNetDeployer/K8sHook.cs @@ -6,10 +6,12 @@ namespace CodexNetDeployer public class K8sHook : IK8sHooks { private readonly string testsTypeLabel; + private readonly Dictionary metadata; - public K8sHook(string testsTypeLabel) + public K8sHook(string testsTypeLabel, Dictionary metadata) { this.testsTypeLabel = testsTypeLabel; + this.metadata = metadata; } public void OnContainersStarted(RunningContainers rc) @@ -25,6 +27,11 @@ namespace CodexNetDeployer recipe.PodLabels.Add("tests-type", testsTypeLabel); recipe.PodLabels.Add("runid", NameUtils.GetRunId()); recipe.PodLabels.Add("testid", NameUtils.GetTestId()); + + foreach (var pair in metadata) + { + recipe.PodLabels.Add(pair.Key, pair.Value); + } } } }