Restores pod labels in deployer. Adds metrics assert for twoclient continuous test.

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -2,7 +2,6 @@
 using GethPlugin;
 using KubernetesWorkflow;
 using Logging;
-using Nethereum.Contracts;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -54,7 +53,7 @@ namespace CodexContractsPlugin
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(tools.GetLog());
-                workflow.DownloadContainerLog(container, logHandler, null);
+                workflow.DownloadContainerLog(container, logHandler, 100);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -109,8 +109,6 @@ namespace CodexPlugin
             {
                 AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
-
-            AddPodLabel("codexid", Image);
         }
 
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -16,6 +16,11 @@ namespace MetricsPlugin
             return Plugin(ci).DeployMetricsCollector(scrapeTargets);
         }
 
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IHasMetricsScrapeTarget scrapeTarget)
+        {
+            return ci.WrapMetricsCollector(metricsContainer, scrapeTarget.MetricsScrapeTarget);
+        }
+
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningContainer metricsContainer, IMetricsScrapeTarget scrapeTarget)
         {
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsContainer, scrapeTarget);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using Logging;
 using System.Globalization;
 
 namespace MetricsPlugin
@@ -7,11 +8,13 @@ namespace MetricsPlugin
     public class MetricsQuery
     {
         private readonly IHttp http;
+        private readonly ILog log;
 
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
             http = tools.CreateHttp(RunningContainer.Address, "api/v1");
+            log = tools.GetLog();
         }
 
         public RunningContainer RunningContainer { get; }
@@ -21,7 +24,7 @@ namespace MetricsPlugin
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
             if (response == null) return null;
 
-            return new Metrics
+            var result = new Metrics
             {
                 Sets = response.data.result.Select(r =>
                 {
@@ -32,20 +35,27 @@ namespace MetricsPlugin
                     };
                 }).ToArray()
             };
+
+            Log(target, metricName, result);
+            return result;
         }
 
         public Metrics? GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
             if (response == null) return null;
-            return MapResponseToMetrics(response);
+            var result = MapResponseToMetrics(response);
+            Log(metricName, result);
+            return result;
         }
 
         public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
             var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
-            return MapResponseToMetrics(response);
+            var result = MapResponseToMetrics(response);
+            Log(target, result);
+            return result;
         }
 
         private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
@@ -135,11 +145,36 @@ namespace MetricsPlugin
             var unixSeconds = ToValue(v);
             return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
         }
+
+        private void Log(IMetricsScrapeTarget target, string metricName, Metrics result)
+        {
+            Log($"{target.Name} '{metricName}' = {result}");
+        }
+
+        private void Log(string metricName, Metrics result)
+        {
+            Log($"'{metricName}' = {result}");
+        }
+
+        private void Log(IMetricsScrapeTarget target, Metrics result)
+        {
+            Log($"{target.Name} => {result}");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
     }
 
     public class Metrics
     {
         public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
+
+        public override string ToString()
+        {
+            return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
+        }
     }
 
     public class MetricsSet
@@ -147,12 +182,22 @@ namespace MetricsPlugin
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
+
+        public override string ToString()
+        {
+            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+        }
     }
 
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
         public double Value { get; set; }
+
+        public override string ToString()
+        {
+            return $"<{Timestamp.ToString("o")}={Value}>";
+        }
     }
 
     public class PrometheusQueryResponse

Tests/CodexContinuousTests/ContinuousTest.cs

@@ -3,6 +3,7 @@ using Core;
 using DistTestCore;
 using FileUtils;
 using Logging;
+using MetricsPlugin;
 
 namespace ContinuousTests
 {
@@ -47,6 +48,20 @@ namespace ContinuousTests
         public CancellationToken CancelToken { get; private set; } = new CancellationToken();
         public NodeRunner NodeRunner { get; private set; } = null!;
 
+        public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target)
+        {
+            return CreateMetricsAccess(target.MetricsScrapeTarget);
+        }
+
+        public IMetricsAccess CreateMetricsAccess(IMetricsScrapeTarget target)
+        {
+            if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment.");
+
+            var entryPointFactory = new EntryPointFactory();
+            var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.CodexDeployment.Metadata.KubeNamespace, Log);
+            return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.CodexDeployment.PrometheusContainer, target);
+        }
+
         public abstract int RequiredNumberOfNodes { get; }
         public abstract TimeSpan RunTestEvery { get; }
         public abstract TestFailMode TestFailMode { get; }

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -7,6 +7,8 @@ namespace ContinuousTests.Tests
 {
     public class TwoClientTest : ContinuousTest
     {
+        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
+
         public override int RequiredNumberOfNodes => 2;
         public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
@@ -17,10 +19,14 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
-            file = FileManager.GenerateFile(80.MB());
+            var size = 80.MB();
+            file = FileManager.GenerateFile(size);
 
-            cid = Nodes[0].UploadFile(file);
+            AssertBytesStoredMetric(size, Nodes[0], () =>
-            Assert.That(cid, Is.Not.Null);
+            {
+                cid = Nodes[0].UploadFile(file);
+                Assert.That(cid, Is.Not.Null);
+            });
         }
 
         [TestMoment(t: 10)]
@@ -30,5 +36,26 @@ namespace ContinuousTests.Tests
 
             file.AssertIsEqual(dl);
         }
+
+        private void AssertBytesStoredMetric(ByteSize uploadedSize, ICodexNode node, Action action)
+        {
+            var lowExpected = uploadedSize.SizeInBytes;
+            var highExpected = uploadedSize.SizeInBytes * 1.2;
+
+            var metrics = CreateMetricsAccess(node);
+            var before = metrics.GetMetric(BytesStoredMetric);
+
+            action();
+
+            Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}.");
+
+            Time.WaitUntil(() =>
+            {
+                var after = metrics.GetMetric(BytesStoredMetric);
+                var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value);
+
+                return highExpected > newBytes && newBytes > lowExpected;
+            });
+        }
     }
 }

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -1,5 +1,8 @@
-using CodexPlugin;
+using CodexContractsPlugin;
-using DistTestCore;
+using CodexPlugin;
+using GethPlugin;
+using KubernetesWorkflow;
+using MetricsPlugin;
 using NUnit.Framework;
 using Utils;
 
@@ -12,24 +15,28 @@ namespace Tests.BasicTests
         [Test]
         public void ContinuousTestSubstitute()
         {
+            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("geth"));
+            var contract = Ci.StartCodexContracts(geth);
+
             var group = AddCodex(5, o => o
-                    //.EnableMetrics()
+                    .EnableMetrics()
-                    //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .EnableMarketplace(geth, contract, 10.Eth(), 100000.TestTokens(), isValidator: true)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithBlockTTL(TimeSpan.FromMinutes(5))
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromSeconds(10))
-                    .WithBlockMaintenanceNumber(10000)
+                    .WithBlockMaintenanceNumber(100)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
                     .WithStorageQuota(1.GB()));
 
             var nodes = group.Cast<CodexNode>().ToArray();
 
+            var rc = Ci.DeployMetricsCollector(nodes);
+
             foreach (var node in nodes)
             {
-                //node.Marketplace.MakeStorageAvailable(
+                node.Marketplace.MakeStorageAvailable(
-                //size: 500.MB(),
+                    size: 500.MB(),
-                //minPricePerBytePerSecond: 1.TestTokens(),
+                    minPriceForTotalSpace: 500.TestTokens(),
-                //maxCollateral: 1024.TestTokens(),
+                    maxCollateral: 1024.TestTokens(),
-                //maxDuration: TimeSpan.FromMinutes(5));
+                    maxDuration: TimeSpan.FromMinutes(5));
             }
 
             var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
@@ -40,7 +47,7 @@ namespace Tests.BasicTests
                 var secondary = allNodes.PickOneRandom();
 
                 Log("Run Test");
-                PerformTest(primary, secondary);
+                PerformTest(primary, secondary, rc);
 
                 Thread.Sleep(TimeSpan.FromSeconds(5));
             }
@@ -81,7 +88,7 @@ namespace Tests.BasicTests
             var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
 
             var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
-            
+
             if (errors.Any())
             {
                 Assert.Fail(string.Join(Environment.NewLine, errors));
@@ -104,26 +111,43 @@ namespace Tests.BasicTests
 
         private ByteSize fileSize = 80.MB();
 
-        private void PerformTest(ICodexNode primary, ICodexNode secondary)
+        private const string BytesStoredMetric = "codexRepostoreBytesUsed";
+
+        private void PerformTest(ICodexNode primary, ICodexNode secondary, RunningContainer rc)
         {
             ScopedTestFiles(() =>
             {
                 var testFile = GenerateTestFile(fileSize);
 
+                var metrics = Ci.WrapMetricsCollector(rc, primary);
+                var beforeBytesStored = metrics.GetMetric(BytesStoredMetric);
+
                 var contentId = primary.UploadFile(testFile);
 
+                var low = fileSize.SizeInBytes;
+                var high = low * 1.2;
+                Log("looking for: " + low + " < " + high);
+
+                Time.WaitUntil(() =>
+                {
+                    var afterBytesStored = metrics.GetMetric(BytesStoredMetric);
+                    var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value);
+
+                    return high > newBytes && newBytes > low;
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2));
+
                 var downloadedFile = secondary.DownloadContent(contentId);
 
                 testFile.AssertIsEqual(downloadedFile);
             });
         }
-        
+
         [Test]
         public void HoldMyBeerTest()
         {
             var blockExpirationTime = TimeSpan.FromMinutes(3);
             var group = AddCodex(3, o => o
-                    //.EnableMetrics()
+                    .EnableMetrics()
                     .WithBlockTTL(blockExpirationTime)
                     .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
                     .WithBlockMaintenanceNumber(10000)

Tests/DistTestCore/TestLifecycle.cs

@@ -11,6 +11,7 @@ namespace DistTestCore
         private const string TestsType = "dist-tests";
         private readonly DateTime testStart;
         private readonly EntryPoint entryPoint;
+        private readonly Dictionary<string, string> metadata;
         private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
@@ -21,6 +22,7 @@ namespace DistTestCore
             testStart = DateTime.UtcNow;
 
             entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
+            metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
 
             log.WriteLogTag();
@@ -78,6 +80,11 @@ namespace DistTestCore
             recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
             recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());
             recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName());
+
+            foreach (var pair in metadata)
+            {
+                recipe.PodLabels.Add(pair.Key, pair.Value);
+            }
         }
 
         public void DownloadAllLogs()

Tools/CodexNetDeployer/Deployer.cs

@@ -85,9 +85,10 @@ namespace CodexNetDeployer
                 retryDelay: TimeSpan.FromSeconds(3),
                 kubernetesNamespace: config.KubeNamespace);
 
-            configuration.Hooks = new K8sHook(config.TestsTypePodLabel);
+            var result = new EntryPoint(log, configuration, string.Empty);
+            configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata());
 
-            return new EntryPoint(log, configuration, string.Empty);
+            return result;
         }
 
         private RunningContainer? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)

Tools/CodexNetDeployer/K8sHook.cs

@@ -6,10 +6,12 @@ namespace CodexNetDeployer
     public class K8sHook : IK8sHooks
     {
         private readonly string testsTypeLabel;
+        private readonly Dictionary<string, string> metadata;
 
-        public K8sHook(string testsTypeLabel)
+        public K8sHook(string testsTypeLabel, Dictionary<string, string> metadata)
         {
             this.testsTypeLabel = testsTypeLabel;
+            this.metadata = metadata;
         }
 
         public void OnContainersStarted(RunningContainers rc)
@@ -25,6 +27,11 @@ namespace CodexNetDeployer
             recipe.PodLabels.Add("tests-type", testsTypeLabel);
             recipe.PodLabels.Add("runid", NameUtils.GetRunId());
             recipe.PodLabels.Add("testid", NameUtils.GetTestId());
+
+            foreach (var pair in metadata)
+            {
+                recipe.PodLabels.Add(pair.Key, pair.Value);
+            }
         }
     }
 }