Merge branch 'feature/grafana-dashboard' into spike/local-continuous-debug
# Conflicts: # ContinuousTests/ContinuousTestRunner.cs
This commit is contained in:
commit
f2346d5d09
|
@ -84,7 +84,7 @@ namespace CodexNetDeployer
|
|||
var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
|
||||
marketplaceConfig.AccountIndexOverride = i;
|
||||
codexStart.MarketplaceConfig = marketplaceConfig;
|
||||
codexStart.MetricsEnabled = config.RecordMetrics;
|
||||
codexStart.MetricsMode = config.Metrics;
|
||||
|
||||
if (config.BlockTTL != Configuration.SecondsIn1Day)
|
||||
{
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
using ArgsUniform;
|
||||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using DistTestCore.Metrics;
|
||||
|
||||
namespace CodexNetDeployer
|
||||
{
|
||||
|
@ -44,15 +44,13 @@ namespace CodexNetDeployer
|
|||
[Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
|
||||
public int BlockTTL { get; set; } = SecondsIn1Day;
|
||||
|
||||
[Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
|
||||
public bool RecordMetrics { get; set; } = false;
|
||||
[Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
|
||||
public MetricsMode Metrics { get; set; } = MetricsMode.None;
|
||||
|
||||
[Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
|
||||
"set this option to override the label value.")]
|
||||
public string TestsTypePodLabel { get; set; } = "continuous";
|
||||
public string TestsTypePodLabel { get; set; } = "continuous-tests";
|
||||
|
||||
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
|
||||
|
||||
public List<string> Validate()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
|
|
@ -27,7 +27,7 @@ namespace CodexNetDeployer
|
|||
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
|
||||
var setup = new CodexSetup(999, config.CodexLogLevel);
|
||||
setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
|
||||
setup.MetricsEnabled = config.RecordMetrics;
|
||||
setup.MetricsMode = config.Metrics;
|
||||
|
||||
Log("Creating Geth instance and deploying contracts...");
|
||||
var gethStarter = new GethStarter(lifecycle);
|
||||
|
@ -52,9 +52,9 @@ namespace CodexNetDeployer
|
|||
if (container != null) codexContainers.Add(container);
|
||||
}
|
||||
|
||||
var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
|
||||
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
|
||||
|
||||
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
|
||||
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
|
||||
}
|
||||
|
||||
private TestLifecycle CreateTestLifecycle()
|
||||
|
@ -68,20 +68,25 @@ namespace CodexNetDeployer
|
|||
logDebug: false,
|
||||
dataFilesPath: "notUsed",
|
||||
codexLogLevel: config.CodexLogLevel,
|
||||
runnerLocation: config.RunnerLocation,
|
||||
k8sNamespacePrefix: config.KubeNamespace
|
||||
);
|
||||
|
||||
return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
|
||||
}
|
||||
|
||||
private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
|
||||
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
|
||||
{
|
||||
if (!setup.MetricsEnabled) return null;
|
||||
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
|
||||
|
||||
Log("Starting metrics service...");
|
||||
var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
|
||||
return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
|
||||
var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
|
||||
|
||||
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
|
||||
|
||||
Log("Starting dashboard service...");
|
||||
var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer);
|
||||
return (prometheusContainer, grafanaStartInfo);
|
||||
}
|
||||
|
||||
private string? GetKubeConfig(string kubeConfigFile)
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
using ArgsUniform;
|
||||
using CodexNetDeployer;
|
||||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
using DistTestCore.Marketplace;
|
||||
using DistTestCore.Metrics;
|
||||
|
@ -17,11 +16,6 @@ public class Program
|
|||
var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
|
||||
var config = uniformArgs.Parse(true);
|
||||
|
||||
if (args.Any(a => a == "--external"))
|
||||
{
|
||||
config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
|
||||
}
|
||||
|
||||
var errors = config.Validate();
|
||||
if (errors.Any())
|
||||
{
|
||||
|
@ -36,7 +30,8 @@ public class Program
|
|||
$"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
|
||||
$"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
|
||||
$"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
|
||||
$"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
|
||||
$"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
|
||||
$"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
|
||||
|
||||
if (!args.Any(a => a == "-y"))
|
||||
{
|
||||
|
|
|
@ -10,4 +10,4 @@ dotnet run \
|
|||
--max-collateral=1024 \
|
||||
--max-duration=3600000 \
|
||||
--block-ttl=300 \
|
||||
--record-metrics=true
|
||||
--metrics=Dashboard
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
using ArgsUniform;
|
||||
using DistTestCore;
|
||||
using DistTestCore.Codex;
|
||||
|
||||
namespace CodexNetDownloader
|
||||
|
@ -16,7 +15,5 @@ namespace CodexNetDownloader
|
|||
public string KubeConfigFile { get; set; } = "null";
|
||||
|
||||
public CodexDeployment CodexDeployment { get; set; } = null!;
|
||||
|
||||
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,17 +15,12 @@ public class Program
|
|||
var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
|
||||
var config = uniformArgs.Parse(true);
|
||||
|
||||
if (args.Any(a => a == "--external"))
|
||||
{
|
||||
config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
|
||||
}
|
||||
|
||||
config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
|
||||
|
||||
if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
|
||||
|
||||
var k8sFactory = new K8sFactory();
|
||||
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
|
||||
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
|
||||
|
||||
foreach (var container in config.CodexDeployment.CodexContainers)
|
||||
{
|
||||
|
|
|
@ -12,7 +12,7 @@ namespace ContinuousTests
|
|||
return containers.Select(container =>
|
||||
{
|
||||
var address = container.ClusterExternalAddress;
|
||||
if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
|
||||
if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
|
||||
return new CodexAccess(log, container, timeSet, address);
|
||||
}).ToArray();
|
||||
}
|
||||
|
|
|
@ -30,7 +30,7 @@ namespace ContinuousTests
|
|||
|
||||
public CodexDeployment CodexDeployment { get; set; } = null!;
|
||||
|
||||
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
|
||||
public RunnerLocation RunnerLocation { get; set; }
|
||||
}
|
||||
|
||||
public class ConfigLoader
|
||||
|
@ -42,10 +42,7 @@ namespace ContinuousTests
|
|||
var result = uniformArgs.Parse(true);
|
||||
|
||||
result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
|
||||
if (args.Any(a => a == "--external"))
|
||||
{
|
||||
result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
|
||||
}
|
||||
result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
@ -63,7 +60,6 @@ namespace ContinuousTests
|
|||
Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
|
||||
"The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
|
||||
|
||||
|
||||
Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
|
||||
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
|
||||
}
|
||||
|
|
|
@ -60,7 +60,7 @@ namespace ContinuousTests
|
|||
if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
|
||||
|
||||
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
|
||||
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
|
||||
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
|
||||
lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
|
||||
}
|
||||
|
||||
|
@ -71,7 +71,7 @@ namespace ContinuousTests
|
|||
var path = Path.Combine(config.LogPath, "containers");
|
||||
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
|
||||
|
||||
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
|
||||
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
|
||||
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken);
|
||||
|
||||
taskFactory.Run(downloader.Run);
|
||||
|
|
|
@ -6,7 +6,7 @@ namespace ContinuousTests
|
|||
{
|
||||
public class K8sFactory
|
||||
{
|
||||
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
|
||||
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
|
||||
{
|
||||
var kubeConfig = GetKubeConfig(kubeConfigFile);
|
||||
var lifecycleConfig = new DistTestCore.Configuration
|
||||
|
@ -16,7 +16,6 @@ namespace ContinuousTests
|
|||
logDebug: false,
|
||||
dataFilesPath: dataFilePath,
|
||||
codexLogLevel: CodexLogLevel.Debug,
|
||||
runnerLocation: runnerLocation,
|
||||
k8sNamespacePrefix: customNamespace
|
||||
);
|
||||
|
||||
|
|
|
@ -91,7 +91,7 @@ namespace ContinuousTests
|
|||
|
||||
private TestLifecycle CreateTestLifecycle()
|
||||
{
|
||||
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
|
||||
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -138,7 +138,7 @@ namespace ContinuousTests
|
|||
private void DownloadClusterLogs()
|
||||
{
|
||||
var k8sFactory = new K8sFactory();
|
||||
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
|
||||
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
|
||||
|
||||
foreach (var container in config.CodexDeployment.CodexContainers)
|
||||
{
|
||||
|
@ -221,7 +221,7 @@ namespace ContinuousTests
|
|||
private DistTestCore.Configuration CreateFileManagerConfiguration()
|
||||
{
|
||||
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
|
||||
CodexLogLevel.Error, config.RunnerLocation, string.Empty);
|
||||
CodexLogLevel.Error, string.Empty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -51,7 +51,7 @@ namespace DistTestCore.Codex
|
|||
{
|
||||
AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
|
||||
}
|
||||
if (config.MetricsEnabled)
|
||||
if (config.MetricsMode != Metrics.MetricsMode.None)
|
||||
{
|
||||
AddEnvVar("CODEX_METRICS", "true");
|
||||
AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
|
||||
|
|
|
@ -5,17 +5,19 @@ namespace DistTestCore.Codex
|
|||
{
|
||||
public class CodexDeployment
|
||||
{
|
||||
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
|
||||
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
|
||||
{
|
||||
GethStartResult = gethStartResult;
|
||||
CodexContainers = codexContainers;
|
||||
PrometheusContainer = prometheusContainer;
|
||||
GrafanaStartInfo = grafanaStartInfo;
|
||||
Metadata = metadata;
|
||||
}
|
||||
|
||||
public GethStartResult GethStartResult { get; }
|
||||
public RunningContainer[] CodexContainers { get; }
|
||||
public RunningContainer? PrometheusContainer { get; }
|
||||
public GrafanaStartInfo? GrafanaStartInfo { get; }
|
||||
public DeploymentMetadata Metadata { get; }
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
using DistTestCore.Marketplace;
|
||||
using DistTestCore.Metrics;
|
||||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Codex
|
||||
|
@ -14,7 +15,7 @@ namespace DistTestCore.Codex
|
|||
public Location Location { get; set; }
|
||||
public CodexLogLevel LogLevel { get; }
|
||||
public ByteSize? StorageQuota { get; set; }
|
||||
public bool MetricsEnabled { get; set; }
|
||||
public MetricsMode MetricsMode { get; set; }
|
||||
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
|
||||
public string? BootstrapSpr { get; set; }
|
||||
public int? BlockTTL { get; set; }
|
||||
|
|
|
@ -59,7 +59,7 @@ namespace DistTestCore
|
|||
|
||||
public ICodexSetup EnableMetrics()
|
||||
{
|
||||
MetricsEnabled = true;
|
||||
MetricsMode = Metrics.MetricsMode.Record;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
|
@ -68,9 +68,15 @@ namespace DistTestCore
|
|||
|
||||
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
|
||||
{
|
||||
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
|
||||
if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
|
||||
|
||||
var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
|
||||
|
||||
if (codexSetup.MetricsMode == MetricsMode.Dashboard)
|
||||
{
|
||||
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First());
|
||||
}
|
||||
|
||||
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
using DistTestCore.Codex;
|
||||
using KubernetesWorkflow;
|
||||
using System.Net.NetworkInformation;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore
|
||||
|
@ -11,8 +12,8 @@ namespace DistTestCore
|
|||
private readonly bool logDebug;
|
||||
private readonly string dataFilesPath;
|
||||
private readonly CodexLogLevel codexLogLevel;
|
||||
private readonly TestRunnerLocation runnerLocation;
|
||||
private readonly string k8sNamespacePrefix;
|
||||
private RunnerLocation? runnerLocation = null;
|
||||
|
||||
public Configuration()
|
||||
{
|
||||
|
@ -21,18 +22,16 @@ namespace DistTestCore
|
|||
logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
|
||||
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
|
||||
codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
|
||||
runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
|
||||
k8sNamespacePrefix = "ct-";
|
||||
}
|
||||
|
||||
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix)
|
||||
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
|
||||
{
|
||||
this.kubeConfigFile = kubeConfigFile;
|
||||
this.logPath = logPath;
|
||||
this.logDebug = logDebug;
|
||||
this.dataFilesPath = dataFilesPath;
|
||||
this.codexLogLevel = codexLogLevel;
|
||||
this.runnerLocation = runnerLocation;
|
||||
this.k8sNamespacePrefix = k8sNamespacePrefix;
|
||||
}
|
||||
|
||||
|
@ -61,14 +60,14 @@ namespace DistTestCore
|
|||
return codexLogLevel;
|
||||
}
|
||||
|
||||
public TestRunnerLocation GetTestRunnerLocation()
|
||||
{
|
||||
return runnerLocation;
|
||||
}
|
||||
|
||||
public Address GetAddress(RunningContainer container)
|
||||
{
|
||||
if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
|
||||
if (runnerLocation == null)
|
||||
{
|
||||
runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
|
||||
}
|
||||
|
||||
if (runnerLocation == RunnerLocation.InternalToCluster)
|
||||
{
|
||||
return container.ClusterInternalAddress;
|
||||
}
|
||||
|
@ -90,9 +89,56 @@ namespace DistTestCore
|
|||
}
|
||||
}
|
||||
|
||||
public enum TestRunnerLocation
|
||||
public enum RunnerLocation
|
||||
{
|
||||
ExternalToCluster,
|
||||
InternalToCluster,
|
||||
}
|
||||
|
||||
public static class RunnerLocationUtils
|
||||
{
|
||||
private static bool alreadyDidThat = false;
|
||||
|
||||
public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
|
||||
{
|
||||
// We want to be sure we don't ping more often than strictly necessary.
|
||||
// If we have already determined the location during this application
|
||||
// lifetime, don't do it again.
|
||||
if (alreadyDidThat) throw new Exception("We already did that.");
|
||||
alreadyDidThat = true;
|
||||
|
||||
if (PingHost(container.Pod.PodInfo.Ip))
|
||||
{
|
||||
return RunnerLocation.InternalToCluster;
|
||||
}
|
||||
if (PingHost(Format(container.ClusterExternalAddress)))
|
||||
{
|
||||
return RunnerLocation.ExternalToCluster;
|
||||
}
|
||||
|
||||
throw new Exception("Unable to determine runner location.");
|
||||
}
|
||||
|
||||
private static string Format(Address host)
|
||||
{
|
||||
return host.Host
|
||||
.Replace("http://", "")
|
||||
.Replace("https://", "");
|
||||
}
|
||||
|
||||
private static bool PingHost(string host)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var pinger = new Ping();
|
||||
PingReply reply = pinger.Send(host);
|
||||
return reply.Status == IPStatus.Success;
|
||||
}
|
||||
catch (PingException)
|
||||
{
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,6 +10,14 @@
|
|||
<PropertyGroup Condition="'$(IsArm64)'=='true'">
|
||||
<DefineConstants>Arm64</DefineConstants>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<None Remove="Metrics\dashboard.json" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Metrics\dashboard.json">
|
||||
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
|
||||
</EmbeddedResource>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
|
|
|
@ -0,0 +1,158 @@
|
|||
using DistTestCore.Metrics;
|
||||
using IdentityModel.Client;
|
||||
using KubernetesWorkflow;
|
||||
using Newtonsoft.Json;
|
||||
using System.Reflection;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class GrafanaStarter : BaseStarter
|
||||
{
|
||||
public GrafanaStarter(TestLifecycle lifecycle)
|
||||
: base(lifecycle)
|
||||
{
|
||||
}
|
||||
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)
|
||||
{
|
||||
LogStart($"Starting dashboard server");
|
||||
|
||||
var grafanaContainer = StartGrafanaContainer();
|
||||
var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
|
||||
|
||||
var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
|
||||
|
||||
Log("Connecting datasource...");
|
||||
AddDataSource(http, prometheusContainer);
|
||||
|
||||
Log("Uploading dashboard configurations...");
|
||||
var jsons = ReadEachDashboardJsonFile();
|
||||
var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
|
||||
|
||||
LogEnd("Dashboard server started.");
|
||||
|
||||
return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
|
||||
}
|
||||
|
||||
private RunningContainer StartGrafanaContainer()
|
||||
{
|
||||
var startupConfig = new StartupConfig();
|
||||
|
||||
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
|
||||
var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
|
||||
if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
|
||||
|
||||
return grafanaContainers.Containers.First();
|
||||
}
|
||||
|
||||
private void AddBasicAuth(HttpClient client)
|
||||
{
|
||||
client.SetBasicAuthentication(
|
||||
GrafanaContainerRecipe.DefaultAdminUser,
|
||||
GrafanaContainerRecipe.DefaultAdminPassword);
|
||||
}
|
||||
|
||||
private static void AddDataSource(Http http, RunningContainer prometheusContainer)
|
||||
{
|
||||
var prometheusAddress = prometheusContainer.ClusterExternalAddress;
|
||||
var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
|
||||
var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
|
||||
{
|
||||
uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
|
||||
name = "CodexPrometheus",
|
||||
type = "prometheus",
|
||||
url = prometheusUrl,
|
||||
access = "proxy",
|
||||
basicAuth = false,
|
||||
jsonData = new GrafanaDataSourceJsonData
|
||||
{
|
||||
httpMethod = "POST"
|
||||
}
|
||||
});
|
||||
|
||||
if (response.message != "Datasource added")
|
||||
{
|
||||
throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
|
||||
}
|
||||
}
|
||||
|
||||
public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
|
||||
{
|
||||
var request = GetDashboardCreateRequest(dashboardJson);
|
||||
var response = http.HttpPostString("dashboards/db", request);
|
||||
var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
|
||||
if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
|
||||
|
||||
var grafanaAddress = grafanaContainer.ClusterExternalAddress;
|
||||
return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
|
||||
}
|
||||
|
||||
private static string[] ReadEachDashboardJsonFile()
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var resourceNames = new[]
|
||||
{
|
||||
"DistTestCore.Metrics.dashboard.json"
|
||||
};
|
||||
|
||||
return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray();
|
||||
}
|
||||
|
||||
private static string GetManifestResource(Assembly assembly, string resourceName)
|
||||
{
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream == null) throw new Exception("Unable to find resource " + resourceName);
|
||||
using var reader = new StreamReader(stream);
|
||||
return reader.ReadToEnd();
|
||||
}
|
||||
|
||||
private static string GetDashboardCreateRequest(string dashboardJson)
|
||||
{
|
||||
return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
|
||||
}
|
||||
}
|
||||
|
||||
public class GrafanaStartInfo
|
||||
{
|
||||
public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
|
||||
{
|
||||
DashboardUrls = dashboardUrls;
|
||||
Container = container;
|
||||
}
|
||||
|
||||
public string[] DashboardUrls { get; }
|
||||
public RunningContainer Container { get; }
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceRequest
|
||||
{
|
||||
public string uid { get; set; } = string.Empty;
|
||||
public string name { get; set; } = string.Empty;
|
||||
public string type { get; set; } = string.Empty;
|
||||
public string url { get; set; } = string.Empty;
|
||||
public string access { get; set; } = string.Empty;
|
||||
public bool basicAuth { get; set; }
|
||||
public GrafanaDataSourceJsonData jsonData { get; set; } = new();
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceResponse
|
||||
{
|
||||
public int id { get; set; }
|
||||
public string message { get; set; } = string.Empty;
|
||||
public string name { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public class GrafanaDataSourceJsonData
|
||||
{
|
||||
public string httpMethod { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public class GrafanaPostDashboardResponse
|
||||
{
|
||||
public int id { get; set; }
|
||||
public string slug { get; set; } = string.Empty;
|
||||
public string status { get; set; } = string.Empty;
|
||||
public string uid { get; set; } = string.Empty;
|
||||
public string url { get; set; } = string.Empty;
|
||||
public int version { get; set; }
|
||||
}
|
||||
}
|
|
@ -12,14 +12,21 @@ namespace DistTestCore
|
|||
private readonly ITimeSet timeSet;
|
||||
private readonly Address address;
|
||||
private readonly string baseUrl;
|
||||
private readonly Action<HttpClient> onClientCreated;
|
||||
private readonly string? logAlias;
|
||||
|
||||
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
|
||||
: this(log, timeSet, address, baseUrl, DoNothing, logAlias)
|
||||
{
|
||||
}
|
||||
|
||||
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
|
||||
{
|
||||
this.log = log;
|
||||
this.timeSet = timeSet;
|
||||
this.address = address;
|
||||
this.baseUrl = baseUrl;
|
||||
this.onClientCreated = onClientCreated;
|
||||
this.logAlias = logAlias;
|
||||
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
|
||||
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
|
||||
|
@ -66,6 +73,22 @@ namespace DistTestCore
|
|||
}, $"HTTP-POST-JSON: {route}");
|
||||
}
|
||||
|
||||
public string HttpPostString(string route, string body)
|
||||
{
|
||||
return Retry(() =>
|
||||
{
|
||||
using var client = GetClient();
|
||||
var url = GetUrl() + route;
|
||||
Log(url, body);
|
||||
var content = new StringContent(body);
|
||||
content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
|
||||
var result = Time.Wait(client.PostAsync(url, content));
|
||||
var str = Time.Wait(result.Content.ReadAsStringAsync());
|
||||
Log(url, str);
|
||||
return str;
|
||||
}, $"HTTP-POST-STRING: {route}");
|
||||
}
|
||||
|
||||
public string HttpPostStream(string route, Stream stream)
|
||||
{
|
||||
return Retry(() =>
|
||||
|
@ -139,7 +162,12 @@ namespace DistTestCore
|
|||
{
|
||||
var client = new HttpClient();
|
||||
client.Timeout = timeSet.HttpCallTimeout();
|
||||
onClientCreated(client);
|
||||
return client;
|
||||
}
|
||||
|
||||
private static void DoNothing(HttpClient client)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,25 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class GrafanaContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
public override string AppName => "grafana";
|
||||
public override string Image => "grafana/grafana-oss:10.0.3";
|
||||
|
||||
public const string DefaultAdminUser = "adminium";
|
||||
public const string DefaultAdminPassword = "passwordium";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
{
|
||||
AddExposedPort(3000);
|
||||
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
|
||||
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
|
||||
|
||||
AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
|
||||
AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public enum MetricsMode
|
||||
{
|
||||
None,
|
||||
Record,
|
||||
Dashboard
|
||||
}
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -22,8 +22,6 @@ namespace DistTestCore
|
|||
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
|
||||
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
|
||||
|
||||
LogEnd("Metrics server started.");
|
||||
|
||||
return runningContainers;
|
||||
}
|
||||
|
||||
|
|
|
@ -24,6 +24,7 @@ namespace DistTestCore
|
|||
FileManager = new FileManager(Log, configuration);
|
||||
CodexStarter = new CodexStarter(this);
|
||||
PrometheusStarter = new PrometheusStarter(this);
|
||||
GrafanaStarter = new GrafanaStarter(this);
|
||||
GethStarter = new GethStarter(this);
|
||||
testStart = DateTime.UtcNow;
|
||||
CodexVersion = null;
|
||||
|
@ -38,6 +39,7 @@ namespace DistTestCore
|
|||
public FileManager FileManager { get; }
|
||||
public CodexStarter CodexStarter { get; }
|
||||
public PrometheusStarter PrometheusStarter { get; }
|
||||
public GrafanaStarter GrafanaStarter { get; }
|
||||
public GethStarter GethStarter { get; }
|
||||
public CodexDebugVersionResponse? CodexVersion { get; private set; }
|
||||
|
||||
|
@ -76,7 +78,8 @@ namespace DistTestCore
|
|||
codexId: GetCodexId(),
|
||||
gethId: new GethContainerRecipe().Image,
|
||||
prometheusId: new PrometheusContainerRecipe().Image,
|
||||
codexContractsId: new CodexContractsContainerRecipe().Image
|
||||
codexContractsId: new CodexContractsContainerRecipe().Image,
|
||||
grafanaId: new GrafanaContainerRecipe().Image
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
@ -40,6 +40,13 @@
|
|||
return p;
|
||||
}
|
||||
|
||||
protected Port AddExposedPort(int number, string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort(number, tag);
|
||||
exposedPorts.Add(p);
|
||||
return p;
|
||||
}
|
||||
|
||||
protected Port AddInternalPort(string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort(tag);
|
||||
|
|
|
@ -25,6 +25,7 @@ namespace KubernetesWorkflow
|
|||
Add("gethid", applicationIds.GethId);
|
||||
Add("prometheusid", applicationIds.PrometheusId);
|
||||
Add("codexcontractsid", applicationIds.CodexContractsId);
|
||||
Add("grafanaid", applicationIds.GrafanaId);
|
||||
}
|
||||
|
||||
public PodLabels GetLabelsForAppName(string appName)
|
||||
|
|
|
@ -7,6 +7,11 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
private NumberSource portNumberSource = new NumberSource(8080);
|
||||
|
||||
public Port CreatePort(int number, string tag)
|
||||
{
|
||||
return new Port(number, tag);
|
||||
}
|
||||
|
||||
public Port CreatePort(string tag)
|
||||
{
|
||||
return new Port(portNumberSource.GetNextNumber(), tag);
|
||||
|
|
|
@ -2,17 +2,19 @@
|
|||
{
|
||||
public class ApplicationIds
|
||||
{
|
||||
public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
|
||||
public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
|
||||
{
|
||||
CodexId = codexId;
|
||||
GethId = gethId;
|
||||
PrometheusId = prometheusId;
|
||||
CodexContractsId = codexContractsId;
|
||||
GrafanaId = grafanaId;
|
||||
}
|
||||
|
||||
public string CodexId { get; }
|
||||
public string GethId { get; }
|
||||
public string PrometheusId { get; }
|
||||
public string CodexContractsId { get; }
|
||||
public string GrafanaId { get; }
|
||||
}
|
||||
}
|
||||
|
|
|
@ -26,6 +26,7 @@ namespace Logging
|
|||
gethid = applicationIds.GethId,
|
||||
prometheusid = applicationIds.PrometheusId,
|
||||
codexcontractsid = applicationIds.CodexContractsId,
|
||||
grafanaid = applicationIds.GrafanaId,
|
||||
category = NameUtils.GetCategoryName(),
|
||||
fixturename = fixtureName,
|
||||
testname = NameUtils.GetTestMethodName(),
|
||||
|
@ -59,6 +60,7 @@ namespace Logging
|
|||
public string gethid { get; set; } = string.Empty;
|
||||
public string prometheusid { get; set; } = string.Empty;
|
||||
public string codexcontractsid { get; set; } = string.Empty;
|
||||
public string grafanaid { get; set; } = string.Empty;
|
||||
public string category { get; set; } = string.Empty;
|
||||
public string fixturename { get; set; } = string.Empty;
|
||||
public string testname { get; set; } = string.Empty;
|
||||
|
|
Loading…
Reference in New Issue