From 4fb0e7c281a51c40288bc9d5ddea4f5f3abcfdb9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 27 Mar 2023 10:27:08 +0200 Subject: [PATCH 01/10] Can enable metrics on codex node startup. --- CodexDistTestCore/CodexNodeContainer.cs | 15 ++++++++++++--- CodexDistTestCore/Config/CodexDockerImage.cs | 15 ++++++++++----- CodexDistTestCore/K8sManager.cs | 6 +++--- CodexDistTestCore/OfflineCodexNodes.cs | 9 +++++++++ LongTests/BasicTests/TestInfraTests.cs | 4 +++- Tests/BasicTests/SimpleTests.cs | 11 +++++++++++ 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index fa0e348..4a1f239 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -2,7 +2,7 @@ { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) { Name = name; ServicePort = servicePort; @@ -12,6 +12,7 @@ DiscoveryPort = discoveryPort; ListenPort = listenPort; DataDir = dataDir; + MetricsPort = metricsPort; } public string Name { get; } @@ -22,6 +23,7 @@ public int DiscoveryPort { get; } public int ListenPort { get; } public string DataDir { get; } + public int MetricsPort { get; } } public class CodexGroupNumberSource @@ -57,7 +59,7 @@ this.groupContainerFactory = groupContainerFactory; } - public CodexNodeContainer CreateNext() + public CodexNodeContainer CreateNext(OfflineCodexNodes offline) { var n = containerNameSource.GetNextNumber(); return new CodexNodeContainer( @@ -68,8 +70,15 @@ containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), listenPort: codexPortSource.GetNextNumber(), - dataDir: $"datadir{n}" + dataDir: $"datadir{n}", + metricsPort: GetMetricsPort(offline) ); } + + private int GetMetricsPort(OfflineCodexNodes offline) + { + if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); + return 0; + } } } diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs index 7cd938e..333ec0d 100644 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ b/CodexDistTestCore/Config/CodexDockerImage.cs @@ -25,12 +25,12 @@ namespace CodexDistTestCore.Config { public List Result { get; } = new List(); - public void Create(OfflineCodexNodes node, CodexNodeContainer environment) + public void Create(OfflineCodexNodes node, CodexNodeContainer container) { - AddVar("API_PORT", environment.ApiPort.ToString()); - AddVar("DATA_DIR", environment.DataDir); - AddVar("DISC_PORT", environment.DiscoveryPort.ToString()); - AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}"); + AddVar("API_PORT", container.ApiPort.ToString()); + AddVar("DATA_DIR", container.DataDir); + AddVar("DISC_PORT", container.DiscoveryPort.ToString()); + AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}"); if (node.BootstrapNode != null) { @@ -45,6 +45,11 @@ namespace CodexDistTestCore.Config { AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!); } + if (node.MetricsEnabled) + { + AddVar("METRICS_ADDR", "0.0.0.0"); + AddVar("METRICS_PORT", container.MetricsPort.ToString()); + } } private void AddVar(string key, string value) diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 22ec144..16a8cba 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -60,18 +60,18 @@ private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { - var containers = CreateContainers(offline.NumberOfNodes); + var containers = CreateContainers(offline); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); onlineCodexNodeGroups.Add(result); return result; } - private CodexNodeContainer[] CreateContainers(int number) + private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline) { var factory = new CodexNodeContainerFactory(codexGroupNumberSource); var containers = new List(); - for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); + for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline)); return containers.ToArray(); } diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index 9092618..f5527af 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -6,6 +6,7 @@ IOfflineCodexNodes WithLogLevel(CodexLogLevel level); IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); + IOfflineCodexNodes EnableMetrics(); ICodexNodeGroup BringOnline(); } @@ -34,12 +35,14 @@ public CodexLogLevel? LogLevel { get; private set; } public IOnlineCodexNode? BootstrapNode { get; private set; } public ByteSize? StorageQuota { get; private set; } + public bool MetricsEnabled { get; private set; } public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) { this.k8SManager = k8SManager; NumberOfNodes = numberOfNodes; Location = Location.Unspecified; + MetricsEnabled = false; } public ICodexNodeGroup BringOnline() @@ -71,6 +74,12 @@ return this; } + public IOfflineCodexNodes EnableMetrics() + { + MetricsEnabled = true; + return this; + } + public string Describe() { var args = string.Join(',', DescribeArgs()); diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 1e2f62e..705b4fc 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -8,7 +8,9 @@ namespace LongTests.BasicTests [Test, UseLongTimeouts] public void TestInfraShouldHave1000AddressSpacesPerPod() { - var group = SetupCodexNodes(1000).BringOnline(); + var group = SetupCodexNodes(1000) + .EnableMetrics() // Increases use of port address space per node. + .BringOnline(); var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 01e671d..80d05b1 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -79,6 +79,17 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } + [Test] + public void MetricsExample() + { + var group = SetupCodexNodes(1) + .EnableMetrics() + .BringOnline(); + + var metrics = BeginGatheringMetrics(group); + + } + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { primary.ConnectToPeer(secondary); From da68fa1de8f501196d0100093fb4aa984e786edf Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 27 Mar 2023 14:49:34 +0200 Subject: [PATCH 02/10] Trying to spin up a prometheus --- CodexDistTestCore/CodexDistTestCore.csproj | 1 + CodexDistTestCore/DistTest.cs | 57 +++++++- CodexDistTestCore/K8sCp.cs | 79 +++++++++++ CodexDistTestCore/K8sManager.cs | 12 ++ CodexDistTestCore/K8sOperations.cs | 52 ++++++- CodexDistTestCore/K8sPrometheusSpecs.cs | 107 +++++++++++++++ CodexDistTestCore/MetricsAccess.cs | 15 +++ CodexDistTestCore/MetricsAggregator.cs | 96 +++++++++++++ CodexDistTestCore/PodLogDownloader.cs | 4 +- Tests/BasicTests/SimpleTests.cs | 150 +++++++++++---------- 10 files changed, 492 insertions(+), 81 deletions(-) create mode 100644 CodexDistTestCore/K8sCp.cs create mode 100644 CodexDistTestCore/K8sPrometheusSpecs.cs create mode 100644 CodexDistTestCore/MetricsAccess.cs create mode 100644 CodexDistTestCore/MetricsAggregator.cs diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index a42e9be..2b63192 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -12,6 +12,7 @@ + diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 1331c6a..45f1be7 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -9,6 +9,7 @@ namespace CodexDistTestCore private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; + private MetricsAggregator metricsAggregator = null!; [OneTimeSetUp] public void GlobalSetup() @@ -48,6 +49,7 @@ namespace CodexDistTestCore fileManager = new FileManager(log); k8sManager = new K8sManager(log, fileManager); + metricsAggregator = new MetricsAggregator(log, k8sManager); } } @@ -57,7 +59,7 @@ namespace CodexDistTestCore try { log.EndTest(); - IncludeLogsOnTestFailure(); + IncludeLogsAndMetricsOnTestFailure(); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } @@ -78,19 +80,57 @@ namespace CodexDistTestCore return new OfflineCodexNodes(k8sManager, numberOfNodes); } - private void IncludeLogsOnTestFailure() + public MetricsAccess GatherMetrics(ICodexNodeGroup group) + { + return GatherMetrics(group.ToArray()); + } + + public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes) + { + Assert.That(nodes.All(n => HasMetricsEnable(n)), + "Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " + + "To use metrics, please use 'EnableMetrics()' when setting up Codex nodes."); + + return metricsAggregator.BeginCollectingMetricsFor(nodes); + } + + public void AssertWithTimeout(Func operation, T isEqualTo, string message) + { + AssertWithTimeout(operation, isEqualTo, TimeSpan.FromMinutes(10), message); + } + + public void AssertWithTimeout(Func operation, T isEqualTo, TimeSpan timeout, string message) + { + var start = DateTime.UtcNow; + + while (true) + { + var result = operation(); + if (result!.Equals(isEqualTo)) return; + if (DateTime.UtcNow - start > timeout) + { + Assert.That(result, Is.EqualTo(isEqualTo), message); + return; + } + + Utils.Sleep(TimeSpan.FromSeconds(10)); + } + } + + private void IncludeLogsAndMetricsOnTestFailure() { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - if (IsDownloadingLogsEnabled()) + if (IsDownloadingLogsAndMetricsEnabled()) { - log.Log("Downloading all CodexNode logs because of test failure..."); + log.Log("Downloading all CodexNode logs and metrics because of test failure..."); k8sManager.ForEachOnlineGroup(DownloadLogs); + metricsAggregator.DownloadAllMetrics(); } else { - log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute."); + log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); } } } @@ -105,11 +145,16 @@ namespace CodexDistTestCore } } - private bool IsDownloadingLogsEnabled() + private bool IsDownloadingLogsAndMetricsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); } + + private bool HasMetricsEnable(IOnlineCodexNode n) + { + return ((OnlineCodexNode)n).Group.Origin.MetricsEnabled; + } } public static class GlobalTestFailure diff --git a/CodexDistTestCore/K8sCp.cs b/CodexDistTestCore/K8sCp.cs new file mode 100644 index 0000000..8faf9ac --- /dev/null +++ b/CodexDistTestCore/K8sCp.cs @@ -0,0 +1,79 @@ +using ICSharpCode.SharpZipLib.Tar; +using k8s; +using System.Text; + +namespace CodexDistTestCore +{ + // From: https://github.com/kubernetes-client/csharp/blob/master/examples/cp/Cp.cs + public class K8sCp + { + private readonly Kubernetes client; + + public K8sCp(Kubernetes client) + { + this.client = client; + } + + public async Task CopyFileToPodAsync(string podName, string @namespace, string containerName, Stream inputFileStream, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken)) + { + var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) => + { + var fileInfo = new FileInfo(destinationFilePath); + try + { + using (var memoryStream = new MemoryStream()) + { + using (var tarOutputStream = new TarOutputStream(memoryStream, Encoding.Default)) + { + tarOutputStream.IsStreamOwner = false; + + var fileSize = inputFileStream.Length; + var entry = TarEntry.CreateTarEntry(fileInfo.Name); + + entry.Size = fileSize; + + tarOutputStream.PutNextEntry(entry); + await inputFileStream.CopyToAsync(tarOutputStream); + tarOutputStream.CloseEntry(); + } + + memoryStream.Position = 0; + + await memoryStream.CopyToAsync(stdIn); + await stdIn.FlushAsync(); + } + + } + catch (Exception ex) + { + throw new IOException($"Copy command failed: {ex.Message}"); + } + + using StreamReader streamReader = new StreamReader(stdError); + while (streamReader.EndOfStream == false) + { + string error = await streamReader.ReadToEndAsync(); + throw new IOException($"Copy command failed: {error}"); + } + }); + + string destinationFolder = GetFolderName(destinationFilePath); + + return await client.NamespacedPodExecAsync( + podName, + @namespace, + containerName, + new string[] { "sh", "-c", $"tar xmf - -C {destinationFolder}" }, + false, + handler, + cancellationToken); + } + + private static string GetFolderName(string filePath) + { + var folderName = Path.GetDirectoryName(filePath); + + return string.IsNullOrEmpty(folderName) ? "." : folderName; + } + } +} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 16a8cba..9d39cad 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -58,6 +58,18 @@ K8s(k => k.FetchPodLog(node, logHandler)); } + public PrometheusInfo BringOnlinePrometheus() + { + PrometheusInfo? info = null; + K8s(k => info = k.BringOnlinePrometheus(codexGroupNumberSource.GetNextServicePort())); + return info!; + } + + public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) + { + K8s(k => k.UploadFileToPod(podName, containerName, fileStream, destinationPath)); + } + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline); diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index a9f1613..be9ba3f 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -1,5 +1,6 @@ using CodexDistTestCore.Config; using k8s; +using k8s.KubeConfigModels; using k8s.Models; using NUnit.Framework; @@ -57,7 +58,30 @@ namespace CodexDistTestCore logHandler.Log(stream); } + public PrometheusInfo BringOnlinePrometheus(int servicePort) + { + EnsureTestNamespace(); + + var spec = new K8sPrometheusSpecs(); + CreatePrometheusDeployment(spec); + CreatePrometheusService(spec, servicePort); + WaitUntilPrometheusOnline(spec); + + return new PrometheusInfo(servicePort, FetchNewPod()); + } + + public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) + { + var cp = new K8sCp(client); + Utils.Wait(cp.CopyFileToPodAsync(podName, K8sCluster.K8sNamespace, containerName, fileStream, destinationPath)); + } + private void FetchPodInfo(CodexNodeGroup online) + { + online.PodInfo = FetchNewPod(); + } + + private PodInfo FetchNewPod() { var pods = client.ListNamespacedPod(K8sNamespace).Items; @@ -65,12 +89,13 @@ namespace CodexDistTestCore Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); var newPod = newPods.Single(); - online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP); + var info = new PodInfo(newPod.Name(), newPod.Status.PodIP); - Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure."); - Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure."); knownPods.Add(newPod.Name()); + return info; } #region Waiting @@ -103,6 +128,16 @@ namespace CodexDistTestCore WaitUntil(() => !IsTestNamespaceOnline()); } + private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) + { + var deploymentName = spec.GetDeploymentName(); + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; + }); + } + private void WaitUntil(Func predicate) { var start = DateTime.UtcNow; @@ -166,6 +201,11 @@ namespace CodexDistTestCore online.Service = null; } + private void CreatePrometheusService(K8sPrometheusSpecs spec, int servicePort) + { + client.CreateNamespacedService(spec.CreatePrometheusService(servicePort), K8sNamespace); + } + #endregion #region Deployment management @@ -232,6 +272,7 @@ namespace CodexDistTestCore Env = dockerImage.CreateEnvironmentVariables(offline, container) }); } + return result; } @@ -242,6 +283,11 @@ namespace CodexDistTestCore online.Deployment = null; } + private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) + { + client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); + } + #endregion #region Namespace management diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs new file mode 100644 index 0000000..c97d60e --- /dev/null +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -0,0 +1,107 @@ +using CodexDistTestCore.Config; +using k8s.Models; + +namespace CodexDistTestCore +{ + public class K8sPrometheusSpecs + { + public const string ContainerName = "dtest-prom"; + public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; + private const string dockerImage = "prom/prometheus:v2.30.3"; + private const string portName = "prom-1"; + + public string GetDeploymentName() + { + return "test-prom"; + } + + public V1Deployment CreatePrometheusDeployment() + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetDeploymentName(), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateSelector() + }, + Spec = new V1PodSpec + { + Containers = new List + { + new V1Container + { + Name = ContainerName, + Image = dockerImage, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = 9090, + Name = portName + } + }, + Command = new List + { + $"--web.enable-lifecycle --config.file={ConfigFilepath}" + }, + } + } + } + } + } + }; + + return deploymentSpec; + } + + public V1Service CreatePrometheusService(int servicePort) + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = "codex-prom-service", + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = CreateSelector(), + Ports = new List + { + new V1ServicePort + { + Name = "prom-service", + Protocol = "TCP", + Port = 9090, + TargetPort = portName, + NodePort = servicePort + } + } + } + }; + + return serviceSpec; + } + + private Dictionary CreateSelector() + { + return new Dictionary { { "test-prom", "dtest-prom" } }; + } + } +} diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs new file mode 100644 index 0000000..33deaca --- /dev/null +++ b/CodexDistTestCore/MetricsAccess.cs @@ -0,0 +1,15 @@ +namespace CodexDistTestCore +{ + public interface IMetricsAccess + { + int GetMostRecentInt(string metricName, IOnlineCodexNode node); + } + + public class MetricsAccess : IMetricsAccess + { + public int GetMostRecentInt(string metricName, IOnlineCodexNode node) + { + return 0; + } + } +} diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs new file mode 100644 index 0000000..273ea0e --- /dev/null +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -0,0 +1,96 @@ +using NUnit.Framework; + +namespace CodexDistTestCore +{ + public class MetricsAggregator + { + private readonly TestLog log; + private readonly K8sManager k8sManager; + private readonly List activeMetricsNodes = new List(); + private PrometheusInfo? activePrometheus; + + public MetricsAggregator(TestLog log, K8sManager k8sManager) + { + this.log = log; + this.k8sManager = k8sManager; + } + + public MetricsAccess BeginCollectingMetricsFor(IOnlineCodexNode[] nodes) + { + EnsurePrometheusPod(); + + AddNewCodexNodes(nodes); + + // Get IPS and ports from all nodes, format prometheus configuration + var config = GeneratePrometheusConfig(); + // Create config file inside prometheus pod + k8sManager.UploadFileToPod( + activePrometheus!.PodInfo.Name, + K8sPrometheusSpecs.ContainerName, + config, + K8sPrometheusSpecs.ConfigFilepath); + + // HTTP POST request to the /-/reload endpoint (when the --web.enable-lifecycle flag is enabled). + + return new MetricsAccess(); + } + + public void DownloadAllMetrics() + { + } + + private void EnsurePrometheusPod() + { + if (activePrometheus != null) return; + activePrometheus = k8sManager.BringOnlinePrometheus(); + } + + private void AddNewCodexNodes(IOnlineCodexNode[] nodes) + { + activeMetricsNodes.AddRange(nodes.Where(n => !activeMetricsNodes.Contains(n)).Cast()); + } + + private Stream GeneratePrometheusConfig() + { + var stream = new MemoryStream(); + using var writer = new StreamWriter(stream); + + writer.WriteLine("global:"); + writer.WriteLine(" scrape_interval: 30s"); + writer.WriteLine(" scrape_timeout: 10s"); + writer.WriteLine(""); + writer.WriteLine("rule_files:"); + writer.WriteLine(" - alert.yml"); + writer.WriteLine(""); + writer.WriteLine("scrape_configs:"); + writer.WriteLine(" - job_name: services"); + writer.WriteLine(" metrics_path: /metrics"); + writer.WriteLine(" static_configs:"); + writer.WriteLine(" - targets:"); + writer.WriteLine(" - 'prometheus:9090'"); + + foreach (var node in activeMetricsNodes) + { + var ip = node.Group.PodInfo!.Ip; + var port = node.Container.ServicePort; + writer.WriteLine($" - '{ip}:{port}'"); + } + + return stream; + } + + + } + + public class PrometheusInfo + { + public PrometheusInfo(int servicePort, PodInfo podInfo) + { + ServicePort = servicePort; + PodInfo = podInfo; + } + + public int ServicePort { get; } + public PodInfo PodInfo { get; } + } +} diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs index ce379d7..4df84d2 100644 --- a/CodexDistTestCore/PodLogDownloader.cs +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -8,9 +8,9 @@ namespace CodexDistTestCore } [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsOnFailureAttribute : PropertyAttribute + public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute { - public DontDownloadLogsOnFailureAttribute() + public DontDownloadLogsAndMetricsOnFailureAttribute() : base(Timing.UseLongTimeoutsKey) { } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 80d05b1..0241add 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -7,89 +7,99 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void GetDebugInfo() - { - var dockerImage = new CodexDockerImage(); + //[Test] + //public void GetDebugInfo() + //{ + // var dockerImage = new CodexDockerImage(); - var node = SetupCodexNodes(1).BringOnline()[0]; + // var node = SetupCodexNodes(1).BringOnline()[0]; - var debugInfo = node.GetDebugInfo(); + // var debugInfo = node.GetDebugInfo(); - Assert.That(debugInfo.spr, Is.Not.Empty); - Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); - } + // Assert.That(debugInfo.spr, Is.Not.Empty); + // Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); + //} - [Test, DontDownloadLogsOnFailure] - public void CanAccessLogs() - { - var node = SetupCodexNodes(1).BringOnline()[0]; + //[Test, DontDownloadLogsAndMetricsOnFailure] + //public void CanAccessLogs() + //{ + // var node = SetupCodexNodes(1).BringOnline()[0]; - var log = node.DownloadLog(); + // var log = node.DownloadLog(); - log.AssertLogContains("Started codex node"); - } - - [Test] - public void OneClientTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = primary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } - - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var secondary = SetupCodexNodes(1).BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; - - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } + // log.AssertLogContains("Started codex node"); + //} [Test] public void MetricsExample() { - var group = SetupCodexNodes(1) - .EnableMetrics() - .BringOnline(); + var group = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - var metrics = BeginGatheringMetrics(group); - + var metrics = GatherMetrics(group); + + var primary = group[0]; + var secondary = group[1]; + primary.ConnectToPeer(secondary); + + Thread.Sleep(10000); + + AssertWithTimeout( + () => metrics.GetMostRecentInt("libp2p_peers", primary), + isEqualTo: 1, + "Number of peers metric was incorrect."); } + //[Test] + //public void OneClientTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; + + // var testFile = GenerateTestFile(1.MB()); + + // var contentId = primary.UploadFile(testFile); + + // var downloadedFile = primary.DownloadContent(contentId); + + // testFile.AssertIsEqual(downloadedFile); + //} + + //[Test] + //public void TwoClientsOnePodTest() + //{ + // var group = SetupCodexNodes(2).BringOnline(); + + // var primary = group[0]; + // var secondary = group[1]; + + // PerformTwoClientTest(primary, secondary); + //} + + //[Test] + //public void TwoClientsTwoPodsTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; + + // var secondary = SetupCodexNodes(1).BringOnline()[0]; + + // PerformTwoClientTest(primary, secondary); + //} + + //[Test] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; + + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; + + // PerformTwoClientTest(primary, secondary); + //} + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { primary.ConnectToPeer(secondary); From 3d1ee8ebdb9bd7a9bd21a3b45dd92620b4c1d34e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 27 Mar 2023 16:24:04 +0200 Subject: [PATCH 03/10] Using customized prometheus image to control configuration. This works. --- CodexDistTestCore/CodexDistTestCore.csproj | 1 - CodexDistTestCore/DistTest.cs | 11 ++- CodexDistTestCore/K8sCp.cs | 79 ---------------------- CodexDistTestCore/K8sManager.cs | 13 ++-- CodexDistTestCore/K8sOperations.cs | 17 ++--- CodexDistTestCore/K8sPrometheusSpecs.cs | 25 +++++-- CodexDistTestCore/MetricsAccess.cs | 55 ++++++++++++++- CodexDistTestCore/MetricsAggregator.cs | 76 +++++++++------------ Tests/BasicTests/SimpleTests.cs | 2 - 9 files changed, 118 insertions(+), 161 deletions(-) delete mode 100644 CodexDistTestCore/K8sCp.cs diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index 2b63192..a42e9be 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -12,7 +12,6 @@ - diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 45f1be7..33af574 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -87,11 +87,13 @@ namespace CodexDistTestCore public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes) { - Assert.That(nodes.All(n => HasMetricsEnable(n)), + var onlineNodes = nodes.Cast().ToArray(); + + Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled), "Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " + "To use metrics, please use 'EnableMetrics()' when setting up Codex nodes."); - return metricsAggregator.BeginCollectingMetricsFor(nodes); + return metricsAggregator.BeginCollectingMetricsFor(onlineNodes); } public void AssertWithTimeout(Func operation, T isEqualTo, string message) @@ -150,11 +152,6 @@ namespace CodexDistTestCore var testProperties = TestContext.CurrentContext.Test.Properties; return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); } - - private bool HasMetricsEnable(IOnlineCodexNode n) - { - return ((OnlineCodexNode)n).Group.Origin.MetricsEnabled; - } } public static class GlobalTestFailure diff --git a/CodexDistTestCore/K8sCp.cs b/CodexDistTestCore/K8sCp.cs deleted file mode 100644 index 8faf9ac..0000000 --- a/CodexDistTestCore/K8sCp.cs +++ /dev/null @@ -1,79 +0,0 @@ -using ICSharpCode.SharpZipLib.Tar; -using k8s; -using System.Text; - -namespace CodexDistTestCore -{ - // From: https://github.com/kubernetes-client/csharp/blob/master/examples/cp/Cp.cs - public class K8sCp - { - private readonly Kubernetes client; - - public K8sCp(Kubernetes client) - { - this.client = client; - } - - public async Task CopyFileToPodAsync(string podName, string @namespace, string containerName, Stream inputFileStream, string destinationFilePath, CancellationToken cancellationToken = default(CancellationToken)) - { - var handler = new ExecAsyncCallback(async (stdIn, stdOut, stdError) => - { - var fileInfo = new FileInfo(destinationFilePath); - try - { - using (var memoryStream = new MemoryStream()) - { - using (var tarOutputStream = new TarOutputStream(memoryStream, Encoding.Default)) - { - tarOutputStream.IsStreamOwner = false; - - var fileSize = inputFileStream.Length; - var entry = TarEntry.CreateTarEntry(fileInfo.Name); - - entry.Size = fileSize; - - tarOutputStream.PutNextEntry(entry); - await inputFileStream.CopyToAsync(tarOutputStream); - tarOutputStream.CloseEntry(); - } - - memoryStream.Position = 0; - - await memoryStream.CopyToAsync(stdIn); - await stdIn.FlushAsync(); - } - - } - catch (Exception ex) - { - throw new IOException($"Copy command failed: {ex.Message}"); - } - - using StreamReader streamReader = new StreamReader(stdError); - while (streamReader.EndOfStream == false) - { - string error = await streamReader.ReadToEndAsync(); - throw new IOException($"Copy command failed: {error}"); - } - }); - - string destinationFolder = GetFolderName(destinationFilePath); - - return await client.NamespacedPodExecAsync( - podName, - @namespace, - containerName, - new string[] { "sh", "-c", $"tar xmf - -C {destinationFolder}" }, - false, - handler, - cancellationToken); - } - - private static string GetFolderName(string filePath) - { - var folderName = Path.GetDirectoryName(filePath); - - return string.IsNullOrEmpty(folderName) ? "." : folderName; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 9d39cad..f86cdbf 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -58,16 +58,13 @@ K8s(k => k.FetchPodLog(node, logHandler)); } - public PrometheusInfo BringOnlinePrometheus() + public PrometheusInfo BringOnlinePrometheus(string config) { - PrometheusInfo? info = null; - K8s(k => info = k.BringOnlinePrometheus(codexGroupNumberSource.GetNextServicePort())); - return info!; - } + var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config); - public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) - { - K8s(k => k.UploadFileToPod(podName, containerName, fileStream, destinationPath)); + PrometheusInfo? info = null; + K8s(k => info = k.BringOnlinePrometheus(spec)); + return info!; } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index be9ba3f..7826c7d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -58,22 +58,15 @@ namespace CodexDistTestCore logHandler.Log(stream); } - public PrometheusInfo BringOnlinePrometheus(int servicePort) + public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) { EnsureTestNamespace(); - var spec = new K8sPrometheusSpecs(); CreatePrometheusDeployment(spec); - CreatePrometheusService(spec, servicePort); + CreatePrometheusService(spec); WaitUntilPrometheusOnline(spec); - return new PrometheusInfo(servicePort, FetchNewPod()); - } - - public void UploadFileToPod(string podName, string containerName, Stream fileStream, string destinationPath) - { - var cp = new K8sCp(client); - Utils.Wait(cp.CopyFileToPodAsync(podName, K8sCluster.K8sNamespace, containerName, fileStream, destinationPath)); + return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } private void FetchPodInfo(CodexNodeGroup online) @@ -201,9 +194,9 @@ namespace CodexDistTestCore online.Service = null; } - private void CreatePrometheusService(K8sPrometheusSpecs spec, int servicePort) + private void CreatePrometheusService(K8sPrometheusSpecs spec) { - client.CreateNamespacedService(spec.CreatePrometheusService(servicePort), K8sNamespace); + client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); } #endregion diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs index c97d60e..1dc8c28 100644 --- a/CodexDistTestCore/K8sPrometheusSpecs.cs +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -7,8 +7,17 @@ namespace CodexDistTestCore { public const string ContainerName = "dtest-prom"; public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; - private const string dockerImage = "prom/prometheus:v2.30.3"; + private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; private const string portName = "prom-1"; + private readonly string config; + + public K8sPrometheusSpecs(int servicePort, string config) + { + ServicePort = servicePort; + this.config = config; + } + + public int ServicePort { get; } public string GetDeploymentName() { @@ -54,10 +63,14 @@ namespace CodexDistTestCore Name = portName } }, - Command = new List + Env = new List { - $"--web.enable-lifecycle --config.file={ConfigFilepath}" - }, + new V1EnvVar + { + Name = "PROM_CONFIG", + Value = config + } + } } } } @@ -68,7 +81,7 @@ namespace CodexDistTestCore return deploymentSpec; } - public V1Service CreatePrometheusService(int servicePort) + public V1Service CreatePrometheusService() { var serviceSpec = new V1Service { @@ -90,7 +103,7 @@ namespace CodexDistTestCore Protocol = "TCP", Port = 9090, TargetPort = portName, - NodePort = servicePort + NodePort = ServicePort } } } diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 33deaca..de8ee01 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Config; + +namespace CodexDistTestCore { public interface IMetricsAccess { @@ -7,9 +9,60 @@ public class MetricsAccess : IMetricsAccess { + private readonly K8sCluster k8sCluster = new K8sCluster(); + private readonly Http http; + + public MetricsAccess(PrometheusInfo prometheusInfo) + { + http = new Http( + k8sCluster.GetIp(), + prometheusInfo.ServicePort, + "api/v1"); + } + public int GetMostRecentInt(string metricName, IOnlineCodexNode node) { + var now = DateTime.UtcNow; + var off = new DateTimeOffset(now); + var nowUnix = off.ToUnixTimeSeconds(); + + var hour = now.AddHours(-1); + var off2 = new DateTimeOffset(hour); + var hourUnix = off2.ToUnixTimeSeconds(); + + var response = http.HttpGetJson($"query_range?query=libp2p_peers&start={hourUnix}&end={nowUnix}&step=100"); + return 0; } } + + public class PrometheusQueryRangeResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryRangeResponseData data { get; set; } = new(); + } + + public class PrometheusQueryRangeResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryRangeResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryRangeResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public ResultEntryValue[] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class ResultEntryValue + { + + } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 273ea0e..594d311 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System.Text; namespace CodexDistTestCore { @@ -6,7 +7,6 @@ namespace CodexDistTestCore { private readonly TestLog log; private readonly K8sManager k8sManager; - private readonly List activeMetricsNodes = new List(); private PrometheusInfo? activePrometheus; public MetricsAggregator(TestLog log, K8sManager k8sManager) @@ -15,71 +15,57 @@ namespace CodexDistTestCore this.k8sManager = k8sManager; } - public MetricsAccess BeginCollectingMetricsFor(IOnlineCodexNode[] nodes) + public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes) { - EnsurePrometheusPod(); + if (activePrometheus != null) + { + Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :("); + throw new InvalidOperationException(); + } - AddNewCodexNodes(nodes); + log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); - // Get IPS and ports from all nodes, format prometheus configuration - var config = GeneratePrometheusConfig(); - // Create config file inside prometheus pod - k8sManager.UploadFileToPod( - activePrometheus!.PodInfo.Name, - K8sPrometheusSpecs.ContainerName, - config, - K8sPrometheusSpecs.ConfigFilepath); + var config = GeneratePrometheusConfig(nodes); + StartPrometheusPod(config); - // HTTP POST request to the /-/reload endpoint (when the --web.enable-lifecycle flag is enabled). - - return new MetricsAccess(); + log.Log("Metrics service started."); + return new MetricsAccess(activePrometheus!); } public void DownloadAllMetrics() { } - private void EnsurePrometheusPod() + private void StartPrometheusPod(string config) { if (activePrometheus != null) return; - activePrometheus = k8sManager.BringOnlinePrometheus(); + activePrometheus = k8sManager.BringOnlinePrometheus(config); } - private void AddNewCodexNodes(IOnlineCodexNode[] nodes) + private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) { - activeMetricsNodes.AddRange(nodes.Where(n => !activeMetricsNodes.Contains(n)).Cast()); - } + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + config += " - 'prometheus:9090'\n"; - private Stream GeneratePrometheusConfig() - { - var stream = new MemoryStream(); - using var writer = new StreamWriter(stream); - - writer.WriteLine("global:"); - writer.WriteLine(" scrape_interval: 30s"); - writer.WriteLine(" scrape_timeout: 10s"); - writer.WriteLine(""); - writer.WriteLine("rule_files:"); - writer.WriteLine(" - alert.yml"); - writer.WriteLine(""); - writer.WriteLine("scrape_configs:"); - writer.WriteLine(" - job_name: services"); - writer.WriteLine(" metrics_path: /metrics"); - writer.WriteLine(" static_configs:"); - writer.WriteLine(" - targets:"); - writer.WriteLine(" - 'prometheus:9090'"); - - foreach (var node in activeMetricsNodes) + foreach (var node in nodes) { var ip = node.Group.PodInfo!.Ip; - var port = node.Container.ServicePort; - writer.WriteLine($" - '{ip}:{port}'"); + var port = node.Container.MetricsPort; + config += $" - '{ip}:{port}'\n"; } - return stream; + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); } - - } public class PrometheusInfo diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 0241add..ba7f15a 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -43,8 +43,6 @@ namespace Tests.BasicTests var secondary = group[1]; primary.ConnectToPeer(secondary); - Thread.Sleep(10000); - AssertWithTimeout( () => metrics.GetMostRecentInt("libp2p_peers", primary), isEqualTo: 1, From 99e5b4c89a5ccc0ecab0b31fe356b9d1a1e772a4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 28 Mar 2023 10:25:48 +0200 Subject: [PATCH 04/10] this works! --- CodexDistTestCore/Config/K8sCluster.cs | 1 + CodexDistTestCore/DistTest.cs | 2 +- CodexDistTestCore/MetricsAccess.cs | 42 +++++----- Tests/BasicTests/SimpleTests.cs | 102 ++++++++++++------------- 4 files changed, 74 insertions(+), 73 deletions(-) diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index 0fd60d0..c43c982 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -18,6 +18,7 @@ namespace CodexDistTestCore.Config { if (config != null) return config; config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + //config = KubernetesClientConfiguration.BuildDefaultConfig(); return config; } diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 33af574..92a1709 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -115,7 +115,7 @@ namespace CodexDistTestCore return; } - Utils.Sleep(TimeSpan.FromSeconds(10)); + Utils.Sleep(TimeSpan.FromSeconds(2)); } } diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index de8ee01..70a6ccb 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -4,7 +4,7 @@ namespace CodexDistTestCore { public interface IMetricsAccess { - int GetMostRecentInt(string metricName, IOnlineCodexNode node); + int? GetMostRecentInt(string metricName, IOnlineCodexNode node); } public class MetricsAccess : IMetricsAccess @@ -20,38 +20,43 @@ namespace CodexDistTestCore "api/v1"); } - public int GetMostRecentInt(string metricName, IOnlineCodexNode node) + public int? GetMostRecentInt(string metricName, IOnlineCodexNode node) { - var now = DateTime.UtcNow; - var off = new DateTimeOffset(now); - var nowUnix = off.ToUnixTimeSeconds(); + var n = (OnlineCodexNode)node; + var pod = n.Group.PodInfo!; - var hour = now.AddHours(-1); - var off2 = new DateTimeOffset(hour); - var hourUnix = off2.ToUnixTimeSeconds(); + var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); + if (response.status != "success") return null; - var response = http.HttpGetJson($"query_range?query=libp2p_peers&start={hourUnix}&end={nowUnix}&step=100"); + var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{n.Container.MetricsPort}"); + if (forNode == null) return null; - return 0; + if (forNode.value == null || forNode.value.Length == 0) return null; + + if (forNode.value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + // [0] = double, timestamp + // [1] = string, value + + return Convert.ToInt32(forNode.value[1]); } } - public class PrometheusQueryRangeResponse + public class PrometheusQueryResponse { public string status { get; set; } = string.Empty; - public PrometheusQueryRangeResponseData data { get; set; } = new(); + public PrometheusQueryResponseData data { get; set; } = new(); } - public class PrometheusQueryRangeResponseData + public class PrometheusQueryResponseData { public string resultType { get; set; } = string.Empty; - public PrometheusQueryRangeResponseDataResultEntry[] result { get; set; } = Array.Empty(); + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); } - public class PrometheusQueryRangeResponseDataResultEntry + public class PrometheusQueryResponseDataResultEntry { public ResultEntryMetric metric { get; set; } = new(); - public ResultEntryValue[] values { get; set; } = Array.Empty(); + public object[] value { get; set; } = Array.Empty(); } public class ResultEntryMetric @@ -60,9 +65,4 @@ namespace CodexDistTestCore public string instance { get; set; } = string.Empty; public string job { get; set; } = string.Empty; } - - public class ResultEntryValue - { - - } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index ba7f15a..74ec0cb 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -7,28 +7,28 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - //[Test] - //public void GetDebugInfo() - //{ - // var dockerImage = new CodexDockerImage(); + [Test] + public void GetDebugInfo() + { + var dockerImage = new CodexDockerImage(); - // var node = SetupCodexNodes(1).BringOnline()[0]; + var node = SetupCodexNodes(1).BringOnline()[0]; - // var debugInfo = node.GetDebugInfo(); + var debugInfo = node.GetDebugInfo(); - // Assert.That(debugInfo.spr, Is.Not.Empty); - // Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); - //} + Assert.That(debugInfo.spr, Is.Not.Empty); + Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); + } - //[Test, DontDownloadLogsAndMetricsOnFailure] - //public void CanAccessLogs() - //{ - // var node = SetupCodexNodes(1).BringOnline()[0]; + [Test, DontDownloadLogsAndMetricsOnFailure] + public void CanAccessLogs() + { + var node = SetupCodexNodes(1).BringOnline()[0]; - // var log = node.DownloadLog(); + var log = node.DownloadLog(); - // log.AssertLogContains("Started codex node"); - //} + log.AssertLogContains("Started codex node"); + } [Test] public void MetricsExample() @@ -49,54 +49,54 @@ namespace Tests.BasicTests "Number of peers metric was incorrect."); } - //[Test] - //public void OneClientTest() - //{ - // var primary = SetupCodexNodes(1).BringOnline()[0]; + [Test] + public void OneClientTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; - // var testFile = GenerateTestFile(1.MB()); + var testFile = GenerateTestFile(1.MB()); - // var contentId = primary.UploadFile(testFile); + var contentId = primary.UploadFile(testFile); - // var downloadedFile = primary.DownloadContent(contentId); + var downloadedFile = primary.DownloadContent(contentId); - // testFile.AssertIsEqual(downloadedFile); - //} + testFile.AssertIsEqual(downloadedFile); + } - //[Test] - //public void TwoClientsOnePodTest() - //{ - // var group = SetupCodexNodes(2).BringOnline(); + [Test] + public void TwoClientsOnePodTest() + { + var group = SetupCodexNodes(2).BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; + var primary = group[0]; + var secondary = group[1]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //public void TwoClientsTwoPodsTest() - //{ - // var primary = SetupCodexNodes(1).BringOnline()[0]; + [Test] + public void TwoClientsTwoPodsTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; - // var secondary = SetupCodexNodes(1).BringOnline()[0]; + var secondary = SetupCodexNodes(1).BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //public void TwoClientsTwoLocationsTest() - //{ - // var primary = SetupCodexNodes(1) - // .At(Location.BensLaptop) - // .BringOnline()[0]; + [Test] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; - // var secondary = SetupCodexNodes(1) - // .At(Location.BensOldGamingMachine) - // .BringOnline()[0]; + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { From 8cbba00407865bb6bd1a20cd651aa019c4ce645b Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 28 Mar 2023 13:43:25 +0200 Subject: [PATCH 05/10] Allows starting of multiple metrics-collecting services --- CodexDistTestCore/Config/K8sCluster.cs | 4 +-- CodexDistTestCore/K8sManager.cs | 4 +-- CodexDistTestCore/K8sPrometheusSpecs.cs | 10 ++++--- CodexDistTestCore/MetricsAccess.cs | 8 +++++- CodexDistTestCore/MetricsAggregator.cs | 19 ++++++------- Tests/BasicTests/SimpleTests.cs | 37 +++++++++++++++++-------- 6 files changed, 51 insertions(+), 31 deletions(-) diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index c43c982..a290193 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -17,8 +17,8 @@ namespace CodexDistTestCore.Config public KubernetesClientConfiguration GetK8sClientConfig() { if (config != null) return config; - config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - //config = KubernetesClientConfiguration.BuildDefaultConfig(); + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); return config; } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index f86cdbf..f9dace8 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -58,9 +58,9 @@ K8s(k => k.FetchPodLog(node, logHandler)); } - public PrometheusInfo BringOnlinePrometheus(string config) + public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) { - var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), config); + var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); PrometheusInfo? info = null; K8s(k => info = k.BringOnlinePrometheus(spec)); diff --git a/CodexDistTestCore/K8sPrometheusSpecs.cs b/CodexDistTestCore/K8sPrometheusSpecs.cs index 1dc8c28..dcda941 100644 --- a/CodexDistTestCore/K8sPrometheusSpecs.cs +++ b/CodexDistTestCore/K8sPrometheusSpecs.cs @@ -11,17 +11,19 @@ namespace CodexDistTestCore private const string portName = "prom-1"; private readonly string config; - public K8sPrometheusSpecs(int servicePort, string config) + public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) { ServicePort = servicePort; + PrometheusNumber = prometheusNumber; this.config = config; } public int ServicePort { get; } + public int PrometheusNumber { get; } public string GetDeploymentName() { - return "test-prom"; + return "test-prom" + PrometheusNumber; } public V1Deployment CreatePrometheusDeployment() @@ -88,7 +90,7 @@ namespace CodexDistTestCore ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = "codex-prom-service", + Name = "codex-prom-service" + PrometheusNumber, NamespaceProperty = K8sCluster.K8sNamespace }, Spec = new V1ServiceSpec @@ -99,7 +101,7 @@ namespace CodexDistTestCore { new V1ServicePort { - Name = "prom-service", + Name = "prom-service" + PrometheusNumber, Protocol = "TCP", Port = 9090, TargetPort = portName, diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 70a6ccb..ca10fc3 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using NUnit.Framework; namespace CodexDistTestCore { @@ -11,18 +12,23 @@ namespace CodexDistTestCore { private readonly K8sCluster k8sCluster = new K8sCluster(); private readonly Http http; + private readonly OnlineCodexNode[] nodes; - public MetricsAccess(PrometheusInfo prometheusInfo) + public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes) { http = new Http( k8sCluster.GetIp(), prometheusInfo.ServicePort, "api/v1"); + this.nodes = nodes; } public int? GetMostRecentInt(string metricName, IOnlineCodexNode node) { var n = (OnlineCodexNode)node; + CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + + "(This CodexNode is tracked by a different instance.)"); + var pod = n.Group.PodInfo!; var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 594d311..2e4696b 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -5,9 +5,10 @@ namespace CodexDistTestCore { public class MetricsAggregator { + private readonly NumberSource prometheusNumberSource = new NumberSource(0); private readonly TestLog log; private readonly K8sManager k8sManager; - private PrometheusInfo? activePrometheus; + private readonly Dictionary activePrometheuses = new Dictionary(); public MetricsAggregator(TestLog log, K8sManager k8sManager) { @@ -17,31 +18,27 @@ namespace CodexDistTestCore public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes) { - if (activePrometheus != null) + var alreadyStartedNodes = nodes.Where(n => activePrometheuses.Values.Any(v => v.Contains(n))); + if (alreadyStartedNodes.Any()) { - Assert.Fail("Incorrect test setup: 'GatherMetrics' may be called only once during a test run. Metrics service targets cannot be changed once started. :("); + Assert.Fail("Incorrect test setup: 'GatherMetrics' was already called on one or more of these OnlineCodexNodes."); throw new InvalidOperationException(); } log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); var config = GeneratePrometheusConfig(nodes); - StartPrometheusPod(config); + var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); + activePrometheuses.Add(prometheus, nodes); log.Log("Metrics service started."); - return new MetricsAccess(activePrometheus!); + return new MetricsAccess(prometheus, nodes); } public void DownloadAllMetrics() { } - private void StartPrometheusPod(string config) - { - if (activePrometheus != null) return; - activePrometheus = k8sManager.BringOnlinePrometheus(config); - } - private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) { var config = ""; diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 74ec0cb..e0dd351 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -39,14 +39,29 @@ namespace Tests.BasicTests var metrics = GatherMetrics(group); + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); + + var metrics2 = GatherMetrics(group2); + var primary = group[0]; var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); AssertWithTimeout( () => metrics.GetMostRecentInt("libp2p_peers", primary), isEqualTo: 1, "Number of peers metric was incorrect."); + + AssertWithTimeout( + () => metrics2.GetMostRecentInt("libp2p_peers", primary2), + isEqualTo: 1, + "Aaa"); } [Test] @@ -84,19 +99,19 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - [Test] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; + //[Test] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { From f0d60493aed9c115dcca982c0338670cd00f6c64 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 29 Mar 2023 10:07:16 +0200 Subject: [PATCH 06/10] Cleaning up metrics support --- CodexDistTestCore/DistTest.cs | 23 ------------- CodexDistTestCore/FileManager.cs | 30 ++++++++-------- CodexDistTestCore/MetricsAccess.cs | 55 ++++++++++++++++++++++++------ CodexDistTestCore/TestLog.cs | 40 +++++++++++++++++++--- CodexDistTestCore/Timing.cs | 16 +++++++++ Tests/BasicTests/SimpleTests.cs | 13 ++----- 6 files changed, 115 insertions(+), 62 deletions(-) diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 92a1709..c1d5c70 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -96,29 +96,6 @@ namespace CodexDistTestCore return metricsAggregator.BeginCollectingMetricsFor(onlineNodes); } - public void AssertWithTimeout(Func operation, T isEqualTo, string message) - { - AssertWithTimeout(operation, isEqualTo, TimeSpan.FromMinutes(10), message); - } - - public void AssertWithTimeout(Func operation, T isEqualTo, TimeSpan timeout, string message) - { - var start = DateTime.UtcNow; - - while (true) - { - var result = operation(); - if (result!.Equals(isEqualTo)) return; - if (DateTime.UtcNow - start > timeout) - { - Assert.That(result, Is.EqualTo(isEqualTo), message); - return; - } - - Utils.Sleep(TimeSpan.FromSeconds(2)); - } - } - private void IncludeLogsAndMetricsOnTestFailure() { var result = TestContext.CurrentContext.Result; diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index 6a2daf7..6fbd55f 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -80,28 +80,30 @@ namespace CodexDistTestCore return info.Length; } - public void AssertIsEqual(TestFile? other) + public void AssertIsEqual(TestFile? actual) { - if (other == null) Assert.Fail("TestFile is null."); - if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - var bytes1 = new byte[FileManager.ChunkSize]; - var bytes2 = new byte[FileManager.ChunkSize]; + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - var read1 = 0; - var read2 = 0; + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; while (true) { - read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize); - read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize); + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - if (read1 == 0 && read2 == 0) return; - Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length."); - CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal."); + if (readExpected == 0 && readActual == 0) return; + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); } } } diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index ca10fc3..fc308b9 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,11 +1,12 @@ using CodexDistTestCore.Config; using NUnit.Framework; +using NUnit.Framework.Constraints; namespace CodexDistTestCore { public interface IMetricsAccess { - int? GetMostRecentInt(string metricName, IOnlineCodexNode node); + void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = ""); } public class MetricsAccess : IMetricsAccess @@ -23,27 +24,59 @@ namespace CodexDistTestCore this.nodes = nodes; } - public int? GetMostRecentInt(string metricName, IOnlineCodexNode node) + public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "") + { + var metricValue = GetMetricWithTimeout(metricName, node); + Assert.That(metricValue, constraint, message); + } + + private double GetMetricWithTimeout(string metricName, IOnlineCodexNode node) + { + var start = DateTime.UtcNow; + + while (true) + { + var mostRecent = GetMostRecent(metricName, node); + if (mostRecent != null) return Convert.ToDouble(mostRecent); + if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + { + Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); + throw new TimeoutException(); + } + + Utils.Sleep(TimeSpan.FromSeconds(2)); + } + } + + private object? GetMostRecent(string metricName, IOnlineCodexNode node) { var n = (OnlineCodexNode)node; CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + "(This CodexNode is tracked by a different instance.)"); - var pod = n.Group.PodInfo!; + var response = GetMetric(metricName); + if (response == null) return null; + var value = GetValueFromResponse(n, response); + if (value == null) return null; + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + return value[1]; + } + + private PrometheusQueryResponse? GetMetric(string metricName) + { var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); if (response.status != "success") return null; + return response; + } - var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{n.Container.MetricsPort}"); + private object[]? GetValueFromResponse(OnlineCodexNode node, PrometheusQueryResponse response) + { + var pod = node.Group.PodInfo!; + var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{node.Container.MetricsPort}"); if (forNode == null) return null; - if (forNode.value == null || forNode.value.Length == 0) return null; - - if (forNode.value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - // [0] = double, timestamp - // [1] = string, value - - return Convert.ToInt32(forNode.value[1]); + return forNode.value; } } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index aca6f1b..658ba66 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -1,5 +1,6 @@ using CodexDistTestCore.Config; using NUnit.Framework; +using System.Xml.Linq; namespace CodexDistTestCore { @@ -39,6 +40,16 @@ namespace CodexDistTestCore Log(result.Message); Log($"{result.StackTrace}"); } + + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + RenameLogFile(); + } + } + + private void RenameLogFile() + { + file.ConcatToFilename("_FAILED"); } public LogFile CreateSubfile() @@ -63,10 +74,15 @@ namespace CodexDistTestCore public class LogFile { + private readonly DateTime now; + private string name; private readonly string filepath; public LogFile(DateTime now, string name) { + this.now = now; + this.name = name; + filepath = Path.Join( LogConfig.LogRoot, $"{now.Year}-{Pad(now.Month)}", @@ -74,12 +90,11 @@ namespace CodexDistTestCore Directory.CreateDirectory(filepath); - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); + GenerateFilename(); } - public string FullFilename { get; } - public string FilenameWithoutPath { get; } + public string FullFilename { get; private set; } = string.Empty; + public string FilenameWithoutPath { get; private set; } = string.Empty; public void Write(string message) { @@ -98,6 +113,17 @@ namespace CodexDistTestCore } } + public void ConcatToFilename(string toAdd) + { + var oldFullName = FullFilename; + + name += toAdd; + + GenerateFilename(); + + File.Move(oldFullName, FullFilename); + } + private static string Pad(int n) { return n.ToString().PadLeft(2, '0'); @@ -107,5 +133,11 @@ namespace CodexDistTestCore { return $"[{DateTime.UtcNow.ToString("u")}]"; } + + private void GenerateFilename() + { + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; + FullFilename = Path.Combine(filepath, FilenameWithoutPath); + } } } diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs index 7ee0edf..cfa9456 100644 --- a/CodexDistTestCore/Timing.cs +++ b/CodexDistTestCore/Timing.cs @@ -40,6 +40,11 @@ namespace CodexDistTestCore return GetTimes().K8sOperationTimeout(); } + public static TimeSpan WaitForMetricTimeout() + { + return GetTimes().WaitForMetricTimeout(); + } + private static ITimeSet GetTimes() { var testProperties = TestContext.CurrentContext.Test.Properties; @@ -55,6 +60,7 @@ namespace CodexDistTestCore TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); + TimeSpan WaitForMetricTimeout(); } public class DefaultTimeSet : ITimeSet @@ -83,6 +89,11 @@ namespace CodexDistTestCore { return TimeSpan.FromMinutes(5); } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromSeconds(30); + } } public class LongTimeSet : ITimeSet @@ -111,5 +122,10 @@ namespace CodexDistTestCore { return TimeSpan.FromMinutes(15); } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromMinutes(5); + } } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index e0dd351..cb5c0f1 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -31,7 +31,7 @@ namespace Tests.BasicTests } [Test] - public void MetricsExample() + public void TwoMetricsExample() { var group = SetupCodexNodes(2) .EnableMetrics() @@ -53,15 +53,8 @@ namespace Tests.BasicTests primary.ConnectToPeer(secondary); primary2.ConnectToPeer(secondary2); - AssertWithTimeout( - () => metrics.GetMostRecentInt("libp2p_peers", primary), - isEqualTo: 1, - "Number of peers metric was incorrect."); - - AssertWithTimeout( - () => metrics2.GetMostRecentInt("libp2p_peers", primary2), - isEqualTo: 1, - "Aaa"); + metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); + metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); } [Test] From 935c2320a783240682e9248b4903e637fbf59c7d Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 29 Mar 2023 11:29:43 +0200 Subject: [PATCH 07/10] Extracts query logic from metricsAccess object --- CodexDistTestCore/MetricsAccess.cs | 80 +++---------- CodexDistTestCore/MetricsAggregator.cs | 8 +- CodexDistTestCore/MetricsQuery.cs | 160 +++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 67 deletions(-) create mode 100644 CodexDistTestCore/MetricsQuery.cs diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index fc308b9..534d478 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,5 +1,4 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; +using NUnit.Framework; using NUnit.Framework.Constraints; namespace CodexDistTestCore @@ -11,33 +10,34 @@ namespace CodexDistTestCore public class MetricsAccess : IMetricsAccess { - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Http http; + private readonly MetricsQuery query; private readonly OnlineCodexNode[] nodes; - public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes) + public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes) { - http = new Http( - k8sCluster.GetIp(), - prometheusInfo.ServicePort, - "api/v1"); + this.query = query; this.nodes = nodes; } public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "") { - var metricValue = GetMetricWithTimeout(metricName, node); + var n = (OnlineCodexNode)node; + CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + + "(This CodexNode is tracked by a different instance.)"); + + var metricSet = GetMetricWithTimeout(metricName, n); + var metricValue = metricSet.Values[0].Value; Assert.That(metricValue, constraint, message); } - private double GetMetricWithTimeout(string metricName, IOnlineCodexNode node) + private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) { var start = DateTime.UtcNow; while (true) { var mostRecent = GetMostRecent(metricName, node); - if (mostRecent != null) return Convert.ToDouble(mostRecent); + if (mostRecent != null) return mostRecent; if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) { Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); @@ -48,60 +48,14 @@ namespace CodexDistTestCore } } - private object? GetMostRecent(string metricName, IOnlineCodexNode node) + private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) { - var n = (OnlineCodexNode)node; - CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + - "(This CodexNode is tracked by a different instance.)"); + var result = query.GetMostRecent(metricName); + if (result == null) return null; - var response = GetMetric(metricName); - if (response == null) return null; - - var value = GetValueFromResponse(n, response); - if (value == null) return null; - if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - return value[1]; - } - - private PrometheusQueryResponse? GetMetric(string metricName) - { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); - if (response.status != "success") return null; - return response; - } - - private object[]? GetValueFromResponse(OnlineCodexNode node, PrometheusQueryResponse response) - { var pod = node.Group.PodInfo!; - var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{node.Container.MetricsPort}"); - if (forNode == null) return null; - if (forNode.value == null || forNode.value.Length == 0) return null; - return forNode.value; + var instance = $"{pod.Ip}:{node.Container.MetricsPort}"; + return result.Sets.SingleOrDefault(r => r.Instance == instance); } } - - public class PrometheusQueryResponse - { - public string status { get; set; } = string.Empty; - public PrometheusQueryResponseData data { get; set; } = new(); - } - - public class PrometheusQueryResponseData - { - public string resultType { get; set; } = string.Empty; - public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); - } - - public class PrometheusQueryResponseDataResultEntry - { - public ResultEntryMetric metric { get; set; } = new(); - public object[] value { get; set; } = Array.Empty(); - } - - public class ResultEntryMetric - { - public string __name__ { get; set; } = string.Empty; - public string instance { get; set; } = string.Empty; - public string job { get; set; } = string.Empty; - } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 2e4696b..2fc7b27 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -8,7 +8,7 @@ namespace CodexDistTestCore private readonly NumberSource prometheusNumberSource = new NumberSource(0); private readonly TestLog log; private readonly K8sManager k8sManager; - private readonly Dictionary activePrometheuses = new Dictionary(); + private readonly Dictionary activePrometheuses = new Dictionary(); public MetricsAggregator(TestLog log, K8sManager k8sManager) { @@ -29,10 +29,11 @@ namespace CodexDistTestCore var config = GeneratePrometheusConfig(nodes); var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); - activePrometheuses.Add(prometheus, nodes); + var query = new MetricsQuery(prometheus); + activePrometheuses.Add(query, nodes); log.Log("Metrics service started."); - return new MetricsAccess(prometheus, nodes); + return new MetricsAccess(query, nodes); } public void DownloadAllMetrics() @@ -51,7 +52,6 @@ namespace CodexDistTestCore config += " metrics_path: /metrics\n"; config += " static_configs:\n"; config += " - targets:\n"; - config += " - 'prometheus:9090'\n"; foreach (var node in nodes) { diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/MetricsQuery.cs new file mode 100644 index 0000000..750aa52 --- /dev/null +++ b/CodexDistTestCore/MetricsQuery.cs @@ -0,0 +1,160 @@ +using CodexDistTestCore.Config; +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsQuery + { + private readonly K8sCluster k8sCluster = new K8sCluster(); + private readonly Http http; + + public MetricsQuery(PrometheusInfo prometheusInfo) + { + http = new Http( + k8sCluster.GetIp(), + prometheusInfo.ServicePort, + "api/v1"); + } + + public Metrics? GetMostRecent(string metricName) + { + var response = GetLastOverTime(metricName); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } +} From f70ce8e8bb00d24a0db379eeacc59bb4aeae6b4d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 30 Mar 2023 10:43:17 +0200 Subject: [PATCH 08/10] implements downloading of metrics on test failure --- CodexDistTestCore/MetricsAccess.cs | 7 +- CodexDistTestCore/MetricsAggregator.cs | 2 + CodexDistTestCore/MetricsDownloader.cs | 97 ++++++++++++++++++++++++++ CodexDistTestCore/MetricsQuery.cs | 62 +++++++++++----- CodexDistTestCore/TestLog.cs | 11 +-- Tests/BasicTests/SimpleTests.cs | 24 ++++--- 6 files changed, 166 insertions(+), 37 deletions(-) create mode 100644 CodexDistTestCore/MetricsDownloader.cs diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 534d478..dbda2a8 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -50,12 +50,9 @@ namespace CodexDistTestCore private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) { - var result = query.GetMostRecent(metricName); + var result = query.GetMostRecent(metricName, node); if (result == null) return null; - - var pod = node.Group.PodInfo!; - var instance = $"{pod.Ip}:{node.Container.MetricsPort}"; - return result.Sets.SingleOrDefault(r => r.Instance == instance); + return result.Sets.LastOrDefault(); } } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 2fc7b27..7b08f3b 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -38,6 +38,8 @@ namespace CodexDistTestCore public void DownloadAllMetrics() { + var download = new MetricsDownloader(log, activePrometheuses); + download.DownloadAllMetrics(); } private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) diff --git a/CodexDistTestCore/MetricsDownloader.cs b/CodexDistTestCore/MetricsDownloader.cs new file mode 100644 index 0000000..22ef526 --- /dev/null +++ b/CodexDistTestCore/MetricsDownloader.cs @@ -0,0 +1,97 @@ +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/MetricsQuery.cs index 750aa52..d8ba684 100644 --- a/CodexDistTestCore/MetricsQuery.cs +++ b/CodexDistTestCore/MetricsQuery.cs @@ -16,9 +16,9 @@ namespace CodexDistTestCore "api/v1"); } - public Metrics? GetMostRecent(string metricName) + public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) { - var response = GetLastOverTime(metricName); + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); if (response == null) return null; return new Metrics @@ -38,23 +38,19 @@ namespace CodexDistTestCore { var response = GetAll(metricName); if (response == null) return null; - - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Instance = r.metric.instance, - Values = MapMultipleValues(r.values) - }; - }).ToArray() - }; + return MapResponseToMetrics(response); } - private PrometheusQueryResponse? GetLastOverTime(string metricName) + public Metrics? GetAllMetricsForNode(OnlineCodexNode node) { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{GetQueryTimeRange()})"); + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); if (response.status != "success") return null; return response; } @@ -66,6 +62,22 @@ namespace CodexDistTestCore return response; } + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + private MetricsSetValue[] MapSingleValue(object[] value) { if (value != null && value.Length > 0) @@ -98,6 +110,17 @@ namespace CodexDistTestCore }; } + private string GetInstanceNameForNode(OnlineCodexNode node) + { + var pod = node.Group.PodInfo!; + return $"{pod.Ip}:{node.Container.MetricsPort}"; + } + + private string GetInstanceStringForNode(OnlineCodexNode node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + private string GetQueryTimeRange() { return "[12h]"; @@ -122,6 +145,7 @@ namespace CodexDistTestCore public class MetricsSet { + public string Name { get; set; } = string.Empty; public string Instance { get; set; } = string.Empty; public MetricsSetValue[] Values { get; set; } = Array.Empty(); } @@ -157,4 +181,10 @@ namespace CodexDistTestCore public string instance { get; set; } = string.Empty; public string job { get; set; } = string.Empty; } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 658ba66..3947678 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -1,6 +1,5 @@ using CodexDistTestCore.Config; using NUnit.Framework; -using System.Xml.Linq; namespace CodexDistTestCore { @@ -52,9 +51,9 @@ namespace CodexDistTestCore file.ConcatToFilename("_FAILED"); } - public LogFile CreateSubfile() + public LogFile CreateSubfile(string ext = "log") { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}"); + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -76,12 +75,14 @@ namespace CodexDistTestCore { private readonly DateTime now; private string name; + private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name) + public LogFile(DateTime now, string name, string ext = "log") { this.now = now; this.name = name; + this.ext = ext; filepath = Path.Join( LogConfig.LogRoot, @@ -136,7 +137,7 @@ namespace CodexDistTestCore private void GenerateFilename() { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; FullFilename = Path.Combine(filepath, FilenameWithoutPath); } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index cb5c0f1..9a8c5c8 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -53,6 +53,8 @@ namespace Tests.BasicTests primary.ConnectToPeer(secondary); primary2.ConnectToPeer(secondary2); + Thread.Sleep(TimeSpan.FromMinutes(5)); + metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); } @@ -92,19 +94,19 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - //[Test] - //public void TwoClientsTwoLocationsTest() - //{ - // var primary = SetupCodexNodes(1) - // .At(Location.BensLaptop) - // .BringOnline()[0]; + [Test] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; - // var secondary = SetupCodexNodes(1) - // .At(Location.BensOldGamingMachine) - // .BringOnline()[0]; + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { From c50cf02f7dbaac266e1c922cc7417f86298bd7dc Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 Mar 2023 10:00:44 +0200 Subject: [PATCH 09/10] Simplifies node-metrics interface --- CodexDistTestCore/CodexNodeLog.cs | 7 ++++++- CodexDistTestCore/DistTest.cs | 20 +------------------- CodexDistTestCore/K8sManager.cs | 19 +++++++++++++++++++ CodexDistTestCore/MetricsAccess.cs | 25 +++++++++++++++---------- CodexDistTestCore/MetricsAggregator.cs | 15 ++++++--------- CodexDistTestCore/OnlineCodexNode.cs | 6 ++++-- Tests/BasicTests/SimpleTests.cs | 12 ++++-------- 7 files changed, 55 insertions(+), 49 deletions(-) diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs index b4cae47..1a0572a 100644 --- a/CodexDistTestCore/CodexNodeLog.cs +++ b/CodexDistTestCore/CodexNodeLog.cs @@ -2,7 +2,12 @@ namespace CodexDistTestCore { - public class CodexNodeLog + public interface ICodexNodeLog + { + void AssertLogContains(string expectedString); + } + + public class CodexNodeLog : ICodexNodeLog { private readonly LogFile logFile; diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index c1d5c70..db32409 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -9,7 +9,6 @@ namespace CodexDistTestCore private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; - private MetricsAggregator metricsAggregator = null!; [OneTimeSetUp] public void GlobalSetup() @@ -49,7 +48,6 @@ namespace CodexDistTestCore fileManager = new FileManager(log); k8sManager = new K8sManager(log, fileManager); - metricsAggregator = new MetricsAggregator(log, k8sManager); } } @@ -80,22 +78,6 @@ namespace CodexDistTestCore return new OfflineCodexNodes(k8sManager, numberOfNodes); } - public MetricsAccess GatherMetrics(ICodexNodeGroup group) - { - return GatherMetrics(group.ToArray()); - } - - public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes) - { - var onlineNodes = nodes.Cast().ToArray(); - - Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled), - "Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " + - "To use metrics, please use 'EnableMetrics()' when setting up Codex nodes."); - - return metricsAggregator.BeginCollectingMetricsFor(onlineNodes); - } - private void IncludeLogsAndMetricsOnTestFailure() { var result = TestContext.CurrentContext.Result; @@ -105,7 +87,7 @@ namespace CodexDistTestCore { log.Log("Downloading all CodexNode logs and metrics because of test failure..."); k8sManager.ForEachOnlineGroup(DownloadLogs); - metricsAggregator.DownloadAllMetrics(); + k8sManager.DownloadAllMetrics(); } else { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index f9dace8..f92c628 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -14,11 +14,13 @@ private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly TestLog log; private readonly IFileManager fileManager; + private readonly MetricsAggregator metricsAggregator; public K8sManager(TestLog log, IFileManager fileManager) { this.log = log; this.fileManager = fileManager; + metricsAggregator = new MetricsAggregator(log, this); } public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) @@ -29,6 +31,11 @@ log.Log($"{online.Describe()} online."); + if (offline.MetricsEnabled) + { + BringOnlineMetrics(online); + } + return online; } @@ -67,6 +74,18 @@ return info!; } + public void DownloadAllMetrics() + { + metricsAggregator.DownloadAllMetrics(); + } + + private void BringOnlineMetrics(CodexNodeGroup group) + { + var onlineNodes = group.Nodes.Cast().ToArray(); + + metricsAggregator.BeginCollectingMetricsFor(onlineNodes); + } + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline); diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index dbda2a8..feb3fe7 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -5,27 +5,32 @@ namespace CodexDistTestCore { public interface IMetricsAccess { - void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = ""); + void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); + } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } } public class MetricsAccess : IMetricsAccess { private readonly MetricsQuery query; - private readonly OnlineCodexNode[] nodes; + private readonly OnlineCodexNode node; - public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes) + public MetricsAccess(MetricsQuery query, OnlineCodexNode node) { this.query = query; - this.nodes = nodes; + this.node = node; } - public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "") + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") { - var n = (OnlineCodexNode)node; - CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + - "(This CodexNode is tracked by a different instance.)"); - - var metricSet = GetMetricWithTimeout(metricName, n); + var metricSet = GetMetricWithTimeout(metricName, node); var metricValue = metricSet.Values[0].Value; Assert.That(metricValue, constraint, message); } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 7b08f3b..59e884d 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -16,15 +16,8 @@ namespace CodexDistTestCore this.k8sManager = k8sManager; } - public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes) + public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes) { - var alreadyStartedNodes = nodes.Where(n => activePrometheuses.Values.Any(v => v.Contains(n))); - if (alreadyStartedNodes.Any()) - { - Assert.Fail("Incorrect test setup: 'GatherMetrics' was already called on one or more of these OnlineCodexNodes."); - throw new InvalidOperationException(); - } - log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); var config = GeneratePrometheusConfig(nodes); @@ -33,7 +26,11 @@ namespace CodexDistTestCore activePrometheuses.Add(query, nodes); log.Log("Metrics service started."); - return new MetricsAccess(query, nodes); + + foreach(var node in nodes) + { + node.Metrics = new MetricsAccess(query, node); + } } public void DownloadAllMetrics() diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index f8cf90f..a9cf511 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -9,7 +9,8 @@ namespace CodexDistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); - CodexNodeLog DownloadLog(); + ICodexNodeLog DownloadLog(); + IMetricsAccess Metrics { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -30,6 +31,7 @@ namespace CodexDistTestCore public CodexNodeContainer Container { get; } public CodexNodeGroup Group { get; internal set; } = null!; + public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); public string GetName() { @@ -80,7 +82,7 @@ namespace CodexDistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - public CodexNodeLog DownloadLog() + public ICodexNodeLog DownloadLog() { return Group.DownloadLog(this); } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 9a8c5c8..efb410a 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -37,13 +37,9 @@ namespace Tests.BasicTests .EnableMetrics() .BringOnline(); - var metrics = GatherMetrics(group); - var group2 = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var metrics2 = GatherMetrics(group2); + .EnableMetrics() + .BringOnline(); var primary = group[0]; var secondary = group[1]; @@ -55,8 +51,8 @@ namespace Tests.BasicTests Thread.Sleep(TimeSpan.FromMinutes(5)); - metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); - metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); + primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); } [Test] From 6af0890571b1fb196b63f27da91567c2b06084d3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 Mar 2023 10:37:55 +0200 Subject: [PATCH 10/10] Ignores two-location test for now --- Tests/BasicTests/SimpleTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index efb410a..ae72183 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -91,6 +91,7 @@ namespace Tests.BasicTests } [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] public void TwoClientsTwoLocationsTest() { var primary = SetupCodexNodes(1)