diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index e790320..859defd 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -85,7 +85,7 @@ namespace CodexNetDeployer if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null); Log("Starting dashboard service..."); - var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer); + var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup); return (prometheusContainer, grafanaStartInfo); } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 3c35e42..14278c7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -74,7 +74,7 @@ namespace DistTestCore if (codexSetup.MetricsMode == MetricsMode.Dashboard) { - lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); + lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup); } return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 61bb17d..679122d 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -8,11 +8,14 @@ namespace DistTestCore { public class GrafanaStarter : BaseStarter { + private const string StorageQuotaThresholdReplaceToken = "\"\""; + private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"\""; + public GrafanaStarter(TestLifecycle lifecycle) : base(lifecycle) { } - public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer) + public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup) { LogStart($"Starting dashboard server"); @@ -25,7 +28,7 @@ namespace DistTestCore AddDataSource(http, prometheusContainer); Log("Uploading dashboard configurations..."); - var jsons = ReadEachDashboardJsonFile(); + var jsons = ReadEachDashboardJsonFile(codexSetup); var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray(); LogEnd("Dashboard server started."); @@ -86,7 +89,7 @@ namespace DistTestCore return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; } - private static string[] ReadEachDashboardJsonFile() + private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup) { var assembly = Assembly.GetExecutingAssembly(); var resourceNames = new[] @@ -94,15 +97,45 @@ namespace DistTestCore "DistTestCore.Metrics.dashboard.json" }; - return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray(); + return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray(); } - private static string GetManifestResource(Assembly assembly, string resourceName) + private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup) { using var stream = assembly.GetManifestResourceStream(resourceName); if (stream == null) throw new Exception("Unable to find resource " + resourceName); using var reader = new StreamReader(stream); - return reader.ReadToEnd(); + return ApplyReplacements(reader.ReadToEnd(), codexSetup); + } + + private static string ApplyReplacements(string input, CodexSetup codexSetup) + { + var quotaString = GetQuotaString(codexSetup); + var softMaxString = GetSoftMaxString(codexSetup); + + return input + .Replace(StorageQuotaThresholdReplaceToken, quotaString) + .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString); + } + + private static string GetQuotaString(CodexSetup codexSetup) + { + return GetCodexStorageQuotaInBytes(codexSetup).ToString(); + } + + private static string GetSoftMaxString(CodexSetup codexSetup) + { + var quota = GetCodexStorageQuotaInBytes(codexSetup); + var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing. + return softMax.ToString(); + } + + private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup) + { + if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes; + + // Codex default: 8GB + return 8.GB().SizeInBytes; } private static string GetDashboardCreateRequest(string dashboardJson) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index b14df70..c2c0709 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -148,14 +148,7 @@ namespace DistTestCore private T Retry(Func operation, string description) { - try - { - return operation(); - } - catch (Exception ex) - { - throw new Exception(description, ex); - } + return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 591f0c3..d53d72b 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -431,6 +431,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": "", + "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -453,7 +455,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, "mappings": [], @@ -466,7 +468,7 @@ }, { "color": "red", - "value": 80 + "value": "" } ] } diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 40bfc77..5cc010a 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,5 +1,4 @@ -using ContinuousTests; -using DistTestCore; +using DistTestCore; using NUnit.Framework; using Utils; @@ -12,53 +11,34 @@ namespace Tests.BasicTests [UseLongTimeouts] public void ContinuousTestSubstitute() { - var nodes = new List(); - for (var i = 0; i < 5; i++) + var group = SetupCodexNodes(5, o => o + .EnableMetrics() + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(3.GB())); + + var nodes = group.Cast().ToArray(); + + foreach (var node in nodes) { - nodes.Add((OnlineCodexNode)SetupCodexNode(o => o - .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) - .WithStorageQuota(3.GB()) - )); + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); } - var cts = new CancellationTokenSource(); - var ct = cts.Token; - var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs"); - Directory.CreateDirectory(dlPath); - - var containers = nodes.Select(n => n.CodexAccess.Container).ToArray(); - var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct); - - var logTask = Task.Run(cd.Run); - - try + var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); + while (DateTime.UtcNow < endTime) { - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } + var allNodes = nodes.ToList(); + var primary = allNodes.PickOneRandom(); + var secondary = allNodes.PickOneRandom(); - var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); - while (DateTime.UtcNow < endTime) - { - var allNodes = nodes.ToList(); - var primary = allNodes.PickOneRandom(); - var secondary = allNodes.PickOneRandom(); + Log("Run Test"); + PerformTest(primary, secondary); - Log("Run Test"); - PerformTest(primary, secondary); - - Thread.Sleep(TimeSpan.FromSeconds(5)); - } - } - finally - { - cts.Cancel(); - logTask.Wait(); + Thread.Sleep(TimeSpan.FromSeconds(5)); } } @@ -77,5 +57,42 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); }); } + + [Test] + [UseLongTimeouts] + public void HoldMyBeerTest() + { + var group = SetupCodexNodes(5, o => o + .EnableMetrics() + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(3.GB())); + + var nodes = group.Cast().ToArray(); + + foreach (var node in nodes) + { + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); + } + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + while (DateTime.UtcNow < endTime) + { + foreach (var node in nodes) + { + var file = GenerateTestFile(80.MB()); + var cid = node.UploadFile(file); + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + } + + Thread.Sleep(TimeSpan.FromMinutes(2)); + } + } } }