Merge branch 'feature/grafana-dashboard' into spike/local-continuous-debug

# Conflicts:
#	ContinuousTests/ContinuousTestRunner.cs

CodexNetDeployer/CodexNodeStarter.cs

@@ -84,7 +84,7 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsEnabled = config.RecordMetrics;
+            codexStart.MetricsMode = config.Metrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {

CodexNetDeployer/Configuration.cs

@@ -1,6 +1,6 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
@@ -44,14 +44,12 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
-        public bool RecordMetrics { get; set; } = false;
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
 
         [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
             "set this option to override the label value.")]
-        public string TestsTypePodLabel { get; set; } = "continuous";
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
-       
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
        
         public List<string> Validate()
         {

CodexNetDeployer/Deployer.cs

@@ -27,7 +27,7 @@ namespace CodexNetDeployer
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsEnabled = config.RecordMetrics;
+            setup.MetricsMode = config.Metrics;
 
             Log("Creating Geth instance and deploying contracts...");
             var gethStarter = new GethStarter(lifecycle);
@@ -52,9 +52,9 @@ namespace CodexNetDeployer
                 if (container != null) codexContainers.Add(container);
             }
 
-            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
         }
 
         private TestLifecycle CreateTestLifecycle()
@@ -68,20 +68,25 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                runnerLocation: config.RunnerLocation,
                 k8sNamespacePrefix: config.KubeNamespace
             );
 
             return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
         }
 
-        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
         {
-            if (!setup.MetricsEnabled) return null;
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
 
             Log("Starting metrics service...");
             var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
-            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer);
+            return (prometheusContainer, grafanaStartInfo);
         }
 
         private string? GetKubeConfig(string kubeConfigFile)

CodexNetDeployer/Program.cs

@@ -1,6 +1,5 @@
 using ArgsUniform;
 using CodexNetDeployer;
-using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -17,11 +16,6 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -36,7 +30,8 @@ public class Program
             $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
             $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
             $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -10,4 +10,4 @@ dotnet run \
     --max-collateral=1024 \
     --max-duration=3600000 \
     --block-ttl=300 \
-    --record-metrics=true
+    --metrics=Dashboard

CodexNetDownloader/Configuration.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -16,7 +15,5 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
-
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,17 +15,12 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -30,7 +30,7 @@ namespace ContinuousTests
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        public RunnerLocation RunnerLocation { get; set; }
     }
 
     public class ConfigLoader
@@ -42,10 +42,7 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            if (args.Any(a => a == "--external"))
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
-            {
-                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-            }
 
             return result;
         }
@@ -63,7 +60,6 @@ namespace ContinuousTests
             Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
                 "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
 
-
             Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }

ContinuousTests/ContinuousTestRunner.cs

@@ -60,7 +60,7 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
             lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
         }
 
@@ -71,7 +71,7 @@ namespace ContinuousTests
             var path = Path.Combine(config.LogPath, "containers");
             if (!Directory.Exists(path)) Directory.CreateDirectory(path);
 
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
             var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken);
 
             taskFactory.Run(downloader.Run);

ContinuousTests/K8sFactory.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -16,7 +16,6 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: runnerLocation,
                 k8sNamespacePrefix: customNamespace
             );
 

ContinuousTests/NodeRunner.cs

@@ -91,7 +91,7 @@ namespace ContinuousTests
 
         private TestLifecycle CreateTestLifecycle()
         {
-            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -138,7 +138,7 @@ namespace ContinuousTests
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -221,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, config.RunnerLocation, string.Empty);
+                CodexLogLevel.Error, string.Empty);
         }
     }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -51,7 +51,7 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.MetricsEnabled)
+            if (config.MetricsMode != Metrics.MetricsMode.None)
             {
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");

DistTestCore/Codex/CodexDeployment.cs

@@ -5,17 +5,19 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -14,7 +15,7 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
+        public MetricsMode MetricsMode { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }

DistTestCore/CodexSetup.cs

@@ -59,7 +59,7 @@ namespace DistTestCore
 
         public ICodexSetup EnableMetrics()
         {
-            MetricsEnabled = true;
+            MetricsMode = Metrics.MetricsMode.Record;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -68,9 +68,15 @@ namespace DistTestCore
 
         private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First());
+            }
+
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 

DistTestCore/Configuration.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -11,8 +12,8 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly TestRunnerLocation runnerLocation;
         private readonly string k8sNamespacePrefix;
+        private RunnerLocation? runnerLocation = null;
 
         public Configuration()
         {
@@ -21,18 +22,16 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
             k8sNamespacePrefix = "ct-";
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.runnerLocation = runnerLocation;
             this.k8sNamespacePrefix = k8sNamespacePrefix;
         }
 
@@ -61,14 +60,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
-        public TestRunnerLocation GetTestRunnerLocation()
-        {
-            return runnerLocation;
-        }
-
         public Address GetAddress(RunningContainer container)
         {
-            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -90,9 +89,56 @@ namespace DistTestCore
         }
     }
 
-    public enum TestRunnerLocation
+    public enum RunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
 }

DistTestCore/DistTestCore.csproj

@@ -10,6 +10,14 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,158 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile();
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile()
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return reader.ReadToEnd();
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Http.cs

@@ -12,14 +12,21 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        {
+        }
+
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -66,6 +73,22 @@ namespace DistTestCore
             }, $"HTTP-POST-JSON: {route}");
         }
 
+        public string HttpPostString(string route, string body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
+        }
+
         public string HttpPostStream(string route, Stream stream)
         {
             return Retry(() =>
@@ -139,7 +162,12 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
             return client;
         }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
     }
 }

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/PrometheusStarter.cs

@@ -22,8 +22,6 @@ namespace DistTestCore
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
-            LogEnd("Metrics server started.");
-
             return runningContainers;
         }
 

DistTestCore/TestLifecycle.cs

@@ -24,6 +24,7 @@ namespace DistTestCore
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this);
             PrometheusStarter = new PrometheusStarter(this);
+            GrafanaStarter = new GrafanaStarter(this);
             GethStarter = new GethStarter(this);
             testStart = DateTime.UtcNow;
             CodexVersion = null;
@@ -38,6 +39,7 @@ namespace DistTestCore
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
         public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
@@ -76,7 +78,8 @@ namespace DistTestCore
                 codexId: GetCodexId(),
                 gethId: new GethContainerRecipe().Image,
                 prometheusId: new PrometheusContainerRecipe().Image,
-                codexContractsId: new CodexContractsContainerRecipe().Image
+                codexContractsId: new CodexContractsContainerRecipe().Image,
+                grafanaId: new GrafanaContainerRecipe().Image
             );
         }
 

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -40,6 +40,13 @@
             return p;
         }
 
+        protected Port AddExposedPort(int number, string tag = "")
+        {
+            var p = factory.CreatePort(number, tag);
+            exposedPorts.Add(p);
+            return p;
+        }
+
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);

KubernetesWorkflow/PodLabels.cs

@@ -25,6 +25,7 @@ namespace KubernetesWorkflow
             Add("gethid", applicationIds.GethId);
             Add("prometheusid", applicationIds.PrometheusId);
             Add("codexcontractsid", applicationIds.CodexContractsId);
+            Add("grafanaid", applicationIds.GrafanaId);
         }
 
         public PodLabels GetLabelsForAppName(string appName)

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,6 +7,11 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
+        public Port CreatePort(int number, string tag)
+        {
+            return new Port(number, tag);
+        }
+
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

Logging/ApplicationIds.cs

@@ -2,17 +2,19 @@
 {
     public class ApplicationIds
     {
-        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
         {
             CodexId = codexId;
             GethId = gethId;
             PrometheusId = prometheusId;
             CodexContractsId = codexContractsId;
+            GrafanaId = grafanaId;
         }
 
         public string CodexId { get; }
         public string GethId { get; }
         public string PrometheusId { get; }
         public string CodexContractsId { get; }
+        public string GrafanaId { get; }
     }
 }

Logging/StatusLog.cs

@@ -26,6 +26,7 @@ namespace Logging
                 gethid = applicationIds.GethId,
                 prometheusid = applicationIds.PrometheusId,
                 codexcontractsid = applicationIds.CodexContractsId,
+                grafanaid = applicationIds.GrafanaId,
                 category = NameUtils.GetCategoryName(),
                 fixturename = fixtureName,
                 testname = NameUtils.GetTestMethodName(),
@@ -59,6 +60,7 @@ namespace Logging
         public string gethid { get; set; } = string.Empty;
         public string prometheusid { get; set; } = string.Empty;
         public string codexcontractsid { get; set; } = string.Empty;
+        public string grafanaid { get; set; } = string.Empty;
         public string category { get; set; } = string.Empty;
         public string fixturename { get; set; } = string.Empty;
         public string testname { get; set; } = string.Empty;