diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 0a3b373..836e8f8 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -84,7 +84,7 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; - codexStart.MetricsEnabled = config.RecordMetrics; + codexStart.MetricsMode = config.Metrics; if (config.BlockTTL != Configuration.SecondsIn1Day) { diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 955084a..0257389 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,6 +1,6 @@ using ArgsUniform; -using DistTestCore; using DistTestCore.Codex; +using DistTestCore.Metrics; namespace CodexNetDeployer { @@ -44,15 +44,13 @@ namespace CodexNetDeployer [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")] public int BlockTTL { get; set; } = SecondsIn1Day; - [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] - public bool RecordMetrics { get; set; } = false; + [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")] + public MetricsMode Metrics { get; set; } = MetricsMode.None; [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + "set this option to override the label value.")] - public string TestsTypePodLabel { get; set; } = "continuous"; + public string TestsTypePodLabel { get; set; } = "continuous-tests"; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; - public List Validate() { var errors = new List(); diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index a5e174e..e790320 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -27,7 +27,7 @@ namespace CodexNetDeployer // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. var setup = new CodexSetup(999, config.CodexLogLevel); setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); - setup.MetricsEnabled = config.RecordMetrics; + setup.MetricsMode = config.Metrics; Log("Creating Geth instance and deploying contracts..."); var gethStarter = new GethStarter(lifecycle); @@ -52,9 +52,9 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers); + var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers); - return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); + return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } private TestLifecycle CreateTestLifecycle() @@ -68,20 +68,25 @@ namespace CodexNetDeployer logDebug: false, dataFilesPath: "notUsed", codexLogLevel: config.CodexLogLevel, - runnerLocation: config.RunnerLocation, k8sNamespacePrefix: config.KubeNamespace ); return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } - private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) { - if (!setup.MetricsEnabled) return null; + if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null); Log("Starting metrics service..."); var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) }; - return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); + var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); + + if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null); + + Log("Starting dashboard service..."); + var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer); + return (prometheusContainer, grafanaStartInfo); } private string? GetKubeConfig(string kubeConfigFile) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index d96b25f..da18e01 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,6 +1,5 @@ using ArgsUniform; using CodexNetDeployer; -using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; using DistTestCore.Metrics; @@ -17,11 +16,6 @@ public class Program var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); - if (args.Any(a => a == "--external")) - { - config.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } - var errors = config.Validate(); if (errors.Any()) { @@ -36,7 +30,8 @@ public class Program $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl + $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl + $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl + - $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl); + $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl + + $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl); if (!args.Any(a => a == "-y")) { diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 338b8c1..c0460c8 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -10,4 +10,4 @@ dotnet run \ --max-collateral=1024 \ --max-duration=3600000 \ --block-ttl=300 \ - --record-metrics=true + --metrics=Dashboard diff --git a/CodexNetDownloader/Configuration.cs b/CodexNetDownloader/Configuration.cs index 008dfbf..a42136a 100644 --- a/CodexNetDownloader/Configuration.cs +++ b/CodexNetDownloader/Configuration.cs @@ -1,5 +1,4 @@ using ArgsUniform; -using DistTestCore; using DistTestCore.Codex; namespace CodexNetDownloader @@ -16,7 +15,5 @@ namespace CodexNetDownloader public string KubeConfigFile { get; set; } = "null"; public CodexDeployment CodexDeployment { get; set; } = null!; - - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; } } diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index 195e96b..5744e71 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -15,17 +15,12 @@ public class Program var uniformArgs = new ArgsUniform(PrintHelp, args); var config = uniformArgs.Parse(true); - if (args.Any(a => a == "--external")) - { - config.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } - config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson); if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/ContinuousTests/CodexAccessFactory.cs b/ContinuousTests/CodexAccessFactory.cs index a149017..78e9069 100644 --- a/ContinuousTests/CodexAccessFactory.cs +++ b/ContinuousTests/CodexAccessFactory.cs @@ -12,7 +12,7 @@ namespace ContinuousTests return containers.Select(container => { var address = container.ClusterExternalAddress; - if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress; + if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress; return new CodexAccess(log, container, timeSet, address); }).ToArray(); } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index b0cebbd..da1eaeb 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -30,7 +30,7 @@ namespace ContinuousTests public CodexDeployment CodexDeployment { get; set; } = null!; - public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; + public RunnerLocation RunnerLocation { get; set; } } public class ConfigLoader @@ -42,10 +42,7 @@ namespace ContinuousTests var result = uniformArgs.Parse(true); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); - if (args.Any(a => a == "--external")) - { - result.RunnerLocation = TestRunnerLocation.ExternalToCluster; - } + result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First()); return result; } @@ -63,7 +60,6 @@ namespace ContinuousTests Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl + "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl); - Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 2f06895..61ecbbf 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -60,7 +60,7 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log); lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources(); } @@ -71,7 +71,7 @@ namespace ContinuousTests var path = Path.Combine(config.LogPath, "containers"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); - var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken); taskFactory.Run(downloader.Run); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index ff9b7e9..6ac661e 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -6,7 +6,7 @@ namespace ContinuousTests { public class K8sFactory { - public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) + public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -16,7 +16,6 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: runnerLocation, k8sNamespacePrefix: customNamespace ); diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 4bb335e..5b79bf5 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -91,7 +91,7 @@ namespace ContinuousTests private TestLifecycle CreateTestLifecycle() { - return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); + return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index cff325e..bbc4888 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -138,7 +138,7 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog()); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -221,7 +221,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, config.RunnerLocation, string.Empty); + CodexLogLevel.Error, string.Empty); } } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index d305110..2f1f383 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -51,7 +51,7 @@ namespace DistTestCore.Codex { AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!); } - if (config.MetricsEnabled) + if (config.MetricsMode != Metrics.MetricsMode.None) { AddEnvVar("CODEX_METRICS", "true"); AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 52192dc..b22c77a 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,17 +5,19 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; PrometheusContainer = prometheusContainer; + GrafanaStartInfo = grafanaStartInfo; Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } public RunningContainer? PrometheusContainer { get; } + public GrafanaStartInfo? GrafanaStartInfo { get; } public DeploymentMetadata Metadata { get; } } diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 2a17334..5e72755 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,5 @@ using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; namespace DistTestCore.Codex @@ -14,7 +15,7 @@ namespace DistTestCore.Codex public Location Location { get; set; } public CodexLogLevel LogLevel { get; } public ByteSize? StorageQuota { get; set; } - public bool MetricsEnabled { get; set; } + public MetricsMode MetricsMode { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public string? BootstrapSpr { get; set; } public int? BlockTTL { get; set; } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 0ffb713..4aefd39 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -59,7 +59,7 @@ namespace DistTestCore public ICodexSetup EnableMetrics() { - MetricsEnabled = true; + MetricsMode = Metrics.MetricsMode.Record; return this; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index a1e38b7..3c35e42 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -68,9 +68,15 @@ namespace DistTestCore private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers) { - if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory(); var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers); + + if (codexSetup.MetricsMode == MetricsMode.Dashboard) + { + lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); + } + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index daccad6..84fb766 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using KubernetesWorkflow; +using System.Net.NetworkInformation; using Utils; namespace DistTestCore @@ -11,8 +12,8 @@ namespace DistTestCore private readonly bool logDebug; private readonly string dataFilesPath; private readonly CodexLogLevel codexLogLevel; - private readonly TestRunnerLocation runnerLocation; private readonly string k8sNamespacePrefix; + private RunnerLocation? runnerLocation = null; public Configuration() { @@ -21,18 +22,16 @@ namespace DistTestCore logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); - runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); k8sNamespacePrefix = "ct-"; } - public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix) + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix) { this.kubeConfigFile = kubeConfigFile; this.logPath = logPath; this.logDebug = logDebug; this.dataFilesPath = dataFilesPath; this.codexLogLevel = codexLogLevel; - this.runnerLocation = runnerLocation; this.k8sNamespacePrefix = k8sNamespacePrefix; } @@ -61,14 +60,14 @@ namespace DistTestCore return codexLogLevel; } - public TestRunnerLocation GetTestRunnerLocation() - { - return runnerLocation; - } - public Address GetAddress(RunningContainer container) { - if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster) + if (runnerLocation == null) + { + runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container); + } + + if (runnerLocation == RunnerLocation.InternalToCluster) { return container.ClusterInternalAddress; } @@ -90,9 +89,56 @@ namespace DistTestCore } } - public enum TestRunnerLocation + public enum RunnerLocation { ExternalToCluster, InternalToCluster, } + + public static class RunnerLocationUtils + { + private static bool alreadyDidThat = false; + + public static RunnerLocation DetermineRunnerLocation(RunningContainer container) + { + // We want to be sure we don't ping more often than strictly necessary. + // If we have already determined the location during this application + // lifetime, don't do it again. + if (alreadyDidThat) throw new Exception("We already did that."); + alreadyDidThat = true; + + if (PingHost(container.Pod.PodInfo.Ip)) + { + return RunnerLocation.InternalToCluster; + } + if (PingHost(Format(container.ClusterExternalAddress))) + { + return RunnerLocation.ExternalToCluster; + } + + throw new Exception("Unable to determine runner location."); + } + + private static string Format(Address host) + { + return host.Host + .Replace("http://", "") + .Replace("https://", ""); + } + + private static bool PingHost(string host) + { + try + { + using var pinger = new Ping(); + PingReply reply = pinger.Send(host); + return reply.Status == IPStatus.Success; + } + catch (PingException) + { + } + + return false; + } + } } diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 512d692..94a2271 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -10,6 +10,14 @@ Arm64 + + + + + + Never + + diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs new file mode 100644 index 0000000..61bb17d --- /dev/null +++ b/DistTestCore/GrafanaStarter.cs @@ -0,0 +1,158 @@ +using DistTestCore.Metrics; +using IdentityModel.Client; +using KubernetesWorkflow; +using Newtonsoft.Json; +using System.Reflection; + +namespace DistTestCore +{ + public class GrafanaStarter : BaseStarter + { + public GrafanaStarter(TestLifecycle lifecycle) + : base(lifecycle) + { + } + public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer) + { + LogStart($"Starting dashboard server"); + + var grafanaContainer = StartGrafanaContainer(); + var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer); + + var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth); + + Log("Connecting datasource..."); + AddDataSource(http, prometheusContainer); + + Log("Uploading dashboard configurations..."); + var jsons = ReadEachDashboardJsonFile(); + var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray(); + + LogEnd("Dashboard server started."); + + return new GrafanaStartInfo(dashboardUrls, grafanaContainer); + } + + private RunningContainer StartGrafanaContainer() + { + var startupConfig = new StartupConfig(); + + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); + var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); + if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created."); + + return grafanaContainers.Containers.First(); + } + + private void AddBasicAuth(HttpClient client) + { + client.SetBasicAuthentication( + GrafanaContainerRecipe.DefaultAdminUser, + GrafanaContainerRecipe.DefaultAdminPassword); + } + + private static void AddDataSource(Http http, RunningContainer prometheusContainer) + { + var prometheusAddress = prometheusContainer.ClusterExternalAddress; + var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port; + var response = http.HttpPostJson("datasources", new GrafanaDataSourceRequest + { + uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", + name = "CodexPrometheus", + type = "prometheus", + url = prometheusUrl, + access = "proxy", + basicAuth = false, + jsonData = new GrafanaDataSourceJsonData + { + httpMethod = "POST" + } + }); + + if (response.message != "Datasource added") + { + throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message); + } + } + + public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson) + { + var request = GetDashboardCreateRequest(dashboardJson); + var response = http.HttpPostString("dashboards/db", request); + var jsonResponse = JsonConvert.DeserializeObject(response); + if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard."); + + var grafanaAddress = grafanaContainer.ClusterExternalAddress; + return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; + } + + private static string[] ReadEachDashboardJsonFile() + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceNames = new[] + { + "DistTestCore.Metrics.dashboard.json" + }; + + return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray(); + } + + private static string GetManifestResource(Assembly assembly, string resourceName) + { + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) throw new Exception("Unable to find resource " + resourceName); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + private static string GetDashboardCreateRequest(string dashboardJson) + { + return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; + } + } + + public class GrafanaStartInfo + { + public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container) + { + DashboardUrls = dashboardUrls; + Container = container; + } + + public string[] DashboardUrls { get; } + public RunningContainer Container { get; } + } + + public class GrafanaDataSourceRequest + { + public string uid { get; set; } = string.Empty; + public string name { get; set; } = string.Empty; + public string type { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public string access { get; set; } = string.Empty; + public bool basicAuth { get; set; } + public GrafanaDataSourceJsonData jsonData { get; set; } = new(); + } + + public class GrafanaDataSourceResponse + { + public int id { get; set; } + public string message { get; set; } = string.Empty; + public string name { get; set; } = string.Empty; + } + + public class GrafanaDataSourceJsonData + { + public string httpMethod { get; set; } = string.Empty; + } + + public class GrafanaPostDashboardResponse + { + public int id { get; set; } + public string slug { get; set; } = string.Empty; + public string status { get; set; } = string.Empty; + public string uid { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public int version { get; set; } + } +} diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index ff52b42..b14df70 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -12,14 +12,21 @@ namespace DistTestCore private readonly ITimeSet timeSet; private readonly Address address; private readonly string baseUrl; + private readonly Action onClientCreated; private readonly string? logAlias; public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null) + : this(log, timeSet, address, baseUrl, DoNothing, logAlias) + { + } + + public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action onClientCreated, string? logAlias = null) { this.log = log; this.timeSet = timeSet; this.address = address; this.baseUrl = baseUrl; + this.onClientCreated = onClientCreated; this.logAlias = logAlias; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; @@ -66,6 +73,22 @@ namespace DistTestCore }, $"HTTP-POST-JSON: {route}"); } + public string HttpPostString(string route, string body) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + Log(url, body); + var content = new StringContent(body); + content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); + var result = Time.Wait(client.PostAsync(url, content)); + var str = Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; + }, $"HTTP-POST-STRING: {route}"); + } + public string HttpPostStream(string route, Stream stream) { return Retry(() => @@ -139,7 +162,12 @@ namespace DistTestCore { var client = new HttpClient(); client.Timeout = timeSet.HttpCallTimeout(); + onClientCreated(client); return client; } + + private static void DoNothing(HttpClient client) + { + } } } diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs new file mode 100644 index 0000000..455af39 --- /dev/null +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -0,0 +1,25 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public class GrafanaContainerRecipe : ContainerRecipeFactory + { + public override string AppName => "grafana"; + public override string Image => "grafana/grafana-oss:10.0.3"; + + public const string DefaultAdminUser = "adminium"; + public const string DefaultAdminPassword = "passwordium"; + + protected override void Initialize(StartupConfig startupConfig) + { + AddExposedPort(3000); + + AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); + AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org."); + AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor"); + + AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser); + AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword); + } + } +} diff --git a/DistTestCore/Metrics/MetricsMode.cs b/DistTestCore/Metrics/MetricsMode.cs new file mode 100644 index 0000000..60b4f5e --- /dev/null +++ b/DistTestCore/Metrics/MetricsMode.cs @@ -0,0 +1,9 @@ +namespace DistTestCore.Metrics +{ + public enum MetricsMode + { + None, + Record, + Dashboard + } +} diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json new file mode 100644 index 0000000..591f0c3 --- /dev/null +++ b/DistTestCore/Metrics/dashboard.json @@ -0,0 +1,1843 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "API Upload/Download", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexApiUploads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Uploads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexApiDownloads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Downloads", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "title": "Local Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBlocks", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Number of Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBytesReserved", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Bytes reserved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBytesUsed", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Bytes used", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 8, + "panels": [], + "title": "Block Exchange", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeBlocksSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Blocks sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeBlocksReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Blocks received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantHaveListsSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantBlockListsSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantHaveListsReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantBlockListsReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists received", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "panels": [], + "title": "Purchases", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesPending_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesSubmitted_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Submitted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesStarted_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Started", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesFinished_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Finished", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 44 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesFailed_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Failed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 44 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesCancelled_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Cancelled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 44 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesUnknown_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Unknown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 44 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesError_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Error", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Codex", + "uid": "ca93f344-dd41-48c4-95a7-876c340e3005", + "version": 6, + "weekStart": "" +} \ No newline at end of file diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 1804b77..3b1e49c 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -22,8 +22,6 @@ namespace DistTestCore var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); - LogEnd("Metrics server started."); - return runningContainers; } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index eede32d..260e438 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -24,6 +24,7 @@ namespace DistTestCore FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this); PrometheusStarter = new PrometheusStarter(this); + GrafanaStarter = new GrafanaStarter(this); GethStarter = new GethStarter(this); testStart = DateTime.UtcNow; CodexVersion = null; @@ -38,6 +39,7 @@ namespace DistTestCore public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } + public GrafanaStarter GrafanaStarter { get; } public GethStarter GethStarter { get; } public CodexDebugVersionResponse? CodexVersion { get; private set; } @@ -76,7 +78,8 @@ namespace DistTestCore codexId: GetCodexId(), gethId: new GethContainerRecipe().Image, prometheusId: new PrometheusContainerRecipe().Image, - codexContractsId: new CodexContractsContainerRecipe().Image + codexContractsId: new CodexContractsContainerRecipe().Image, + grafanaId: new GrafanaContainerRecipe().Image ); } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 470bd42..36fff5b 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -40,6 +40,13 @@ return p; } + protected Port AddExposedPort(int number, string tag = "") + { + var p = factory.CreatePort(number, tag); + exposedPorts.Add(p); + return p; + } + protected Port AddInternalPort(string tag = "") { var p = factory.CreatePort(tag); diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs index 60fc332..85b3a98 100644 --- a/KubernetesWorkflow/PodLabels.cs +++ b/KubernetesWorkflow/PodLabels.cs @@ -25,6 +25,7 @@ namespace KubernetesWorkflow Add("gethid", applicationIds.GethId); Add("prometheusid", applicationIds.PrometheusId); Add("codexcontractsid", applicationIds.CodexContractsId); + Add("grafanaid", applicationIds.GrafanaId); } public PodLabels GetLabelsForAppName(string appName) diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index cf1f67b..9644515 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -7,6 +7,11 @@ namespace KubernetesWorkflow { private NumberSource portNumberSource = new NumberSource(8080); + public Port CreatePort(int number, string tag) + { + return new Port(number, tag); + } + public Port CreatePort(string tag) { return new Port(portNumberSource.GetNextNumber(), tag); diff --git a/Logging/ApplicationIds.cs b/Logging/ApplicationIds.cs index d52dc10..7162a36 100644 --- a/Logging/ApplicationIds.cs +++ b/Logging/ApplicationIds.cs @@ -2,17 +2,19 @@ { public class ApplicationIds { - public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId) + public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId) { CodexId = codexId; GethId = gethId; PrometheusId = prometheusId; CodexContractsId = codexContractsId; + GrafanaId = grafanaId; } public string CodexId { get; } public string GethId { get; } public string PrometheusId { get; } public string CodexContractsId { get; } + public string GrafanaId { get; } } } diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs index 96a8cc2..f5d5831 100644 --- a/Logging/StatusLog.cs +++ b/Logging/StatusLog.cs @@ -26,6 +26,7 @@ namespace Logging gethid = applicationIds.GethId, prometheusid = applicationIds.PrometheusId, codexcontractsid = applicationIds.CodexContractsId, + grafanaid = applicationIds.GrafanaId, category = NameUtils.GetCategoryName(), fixturename = fixtureName, testname = NameUtils.GetTestMethodName(), @@ -59,6 +60,7 @@ namespace Logging public string gethid { get; set; } = string.Empty; public string prometheusid { get; set; } = string.Empty; public string codexcontractsid { get; set; } = string.Empty; + public string grafanaid { get; set; } = string.Empty; public string category { get; set; } = string.Empty; public string fixturename { get; set; } = string.Empty; public string testname { get; set; } = string.Empty;