From 3884a6a7f7fd9e7a1a8732b9013239cd8edc9313 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 2 Aug 2023 15:11:27 +0200 Subject: [PATCH 01/31] a --- ContinuousTests/ContinuousLogDownloader.cs | 10 +- ContinuousTests/ContinuousTestRunner.cs | 2 +- DistTestCore/DistTest.cs | 2 +- DistTestCore/Http.cs | 9 +- DistTestCore/OnlineCodexNode.cs | 9 +- Tests/BasicTests/ContinuousSubstitute.cs | 134 +++++++++++++++++++++ Tests/Tests.csproj | 1 + 7 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 Tests/BasicTests/ContinuousSubstitute.cs diff --git a/ContinuousTests/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs index f5e0161..6e387a1 100644 --- a/ContinuousTests/ContinuousLogDownloader.cs +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -7,14 +7,16 @@ namespace ContinuousTests public class ContinuousLogDownloader { private readonly TestLifecycle lifecycle; - private readonly CodexDeployment deployment; + private readonly RunningContainer[] containers; + //private readonly CodexDeployment deployment; private readonly string outputPath; private readonly CancellationToken cancelToken; - public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken) + public ContinuousLogDownloader(TestLifecycle lifecycle, RunningContainer[] containers, string outputPath, CancellationToken cancelToken) { this.lifecycle = lifecycle; - this.deployment = deployment; + this.containers = containers; + //this.deployment = deployment; this.outputPath = outputPath; this.cancelToken = cancelToken; } @@ -37,7 +39,7 @@ namespace ContinuousTests private void UpdateLogs() { - foreach (var container in deployment.CodexContainers) + foreach (var container in containers) { UpdateLog(container); } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 985707a..cbbe777 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -72,7 +72,7 @@ namespace ContinuousTests if (!Directory.Exists(path)) Directory.CreateDirectory(path); var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); - var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); + var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken); taskFactory.Run(downloader.Run); } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 6768618..35d6841 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -179,7 +179,7 @@ namespace DistTestCore return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel()); } - private TestLifecycle Get() + public TestLifecycle Get() { lock (lifecycleLock) { diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 391dbc7..b66ff75 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -125,7 +125,14 @@ namespace DistTestCore private T Retry(Func operation, string description) { - return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description); + try + { + return operation(); + } + catch (Exception ex) + { + throw new Exception(description, ex); + } } private HttpClient GetClient() diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 5a7658c..c220e3c 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -4,6 +4,7 @@ using DistTestCore.Marketplace; using DistTestCore.Metrics; using Logging; using NUnit.Framework; +using Utils; namespace DistTestCore { @@ -66,7 +67,8 @@ namespace DistTestCore { using var fileStream = File.OpenRead(file.Filename); - var logMessage = $"Uploading file {file.Describe()}..."; + var logMessage = $"{CodexAccess.Container.Name} Uploading file {file.Describe()}..."; + Log(logMessage); var response = Stopwatch.Measure(lifecycle.Log, logMessage, () => { return CodexAccess.UploadFile(fileStream); @@ -81,7 +83,8 @@ namespace DistTestCore public TestFile? DownloadContent(ContentId contentId, string fileLabel = "") { - var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; + var logMessage = $"{CodexAccess.Container.Name} Downloading for contentId: '{contentId.Id}'..."; + Log(logMessage); var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel); Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file)); Log($"Downloaded file {file.Describe()} to '{file.Filename}'."); @@ -116,7 +119,7 @@ namespace DistTestCore public void EnsureOnlineGetVersionResponse() { - var debugInfo = CodexAccess.GetDebugInfo(); + var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online"); var nodePeerId = debugInfo.id; var nodeName = CodexAccess.Container.Name; diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs new file mode 100644 index 0000000..8911a27 --- /dev/null +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -0,0 +1,134 @@ +using ContinuousTests; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [TestFixture] + public class ContinuousSubstitute : AutoBootstrapDistTest + { + [Test] + public void ContinuousTestSubstitute() + { + var nodes = new List(); + for (var i = 0; i < 5; i++) + { + nodes.Add((OnlineCodexNode)SetupCodexNode(o => o + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) + .WithStorageQuota(2.GB()) + )); + } + + var cts = new CancellationTokenSource(); + var ct = cts.Token; + var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs"); + Directory.CreateDirectory(dlPath); + + var containers = nodes.Select(n => n.CodexAccess.Container).ToArray(); + var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct); + + var logTask = Task.Run(cd.Run); + + try + { + foreach (var node in nodes) + { + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); + } + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); + while (DateTime.UtcNow < endTime) + { + var allNodes = nodes.ToList(); + var primary = allNodes.PickOneRandom(); + var secondary = allNodes.PickOneRandom(); + + Log("Run Test"); + PerformTest(primary, secondary); + + Thread.Sleep(TimeSpan.FromSeconds(30)); + } + } + finally + { + cts.Cancel(); + logTask.Wait(); + } + } + + [Test] + public void JustWaitFirst() + { + var nodes = new List(); + for (var i = 0; i < 5; i++) + { + nodes.Add((OnlineCodexNode)SetupCodexNode(o => o + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) + .WithStorageQuota(2.GB()) + )); + } + + var cts = new CancellationTokenSource(); + var ct = cts.Token; + var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogsjustwait"); + Directory.CreateDirectory(dlPath); + + var containers = nodes.Select(n => n.CodexAccess.Container).ToArray(); + var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct); + + var logTask = Task.Run(cd.Run); + + try + { + foreach (var node in nodes) + { + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); + } + + Log("Waiting 30 minutes..."); + Thread.Sleep(TimeSpan.FromMinutes(30)); + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); + while (DateTime.UtcNow < endTime) + { + var allNodes = nodes.ToList(); + var primary = allNodes.PickOneRandom(); + var secondary = allNodes.PickOneRandom(); + + Log("Run Test"); + PerformTest(primary, secondary); + + Thread.Sleep(TimeSpan.FromSeconds(30)); + } + } + finally + { + cts.Cancel(); + logTask.Wait(); + } + } + + private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + ScopedTestFiles(() => + { + var testFile = GenerateTestFile(10.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + }); + } + } +} diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 90f1cd6..ab5558b 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -13,6 +13,7 @@ + From ab42fa80041b5d5cf31430cc69cf475dd5a6bb6e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 7 Aug 2023 18:31:13 +0200 Subject: [PATCH 02/31] removes just-wait tests --- Tests/BasicTests/ContinuousSubstitute.cs | 58 +----------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 8911a27..e589df0 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -61,67 +61,11 @@ namespace Tests.BasicTests } } - [Test] - public void JustWaitFirst() - { - var nodes = new List(); - for (var i = 0; i < 5; i++) - { - nodes.Add((OnlineCodexNode)SetupCodexNode(o => o - .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) - .WithStorageQuota(2.GB()) - )); - } - - var cts = new CancellationTokenSource(); - var ct = cts.Token; - var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogsjustwait"); - Directory.CreateDirectory(dlPath); - - var containers = nodes.Select(n => n.CodexAccess.Container).ToArray(); - var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct); - - var logTask = Task.Run(cd.Run); - - try - { - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } - - Log("Waiting 30 minutes..."); - Thread.Sleep(TimeSpan.FromMinutes(30)); - - var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); - while (DateTime.UtcNow < endTime) - { - var allNodes = nodes.ToList(); - var primary = allNodes.PickOneRandom(); - var secondary = allNodes.PickOneRandom(); - - Log("Run Test"); - PerformTest(primary, secondary); - - Thread.Sleep(TimeSpan.FromSeconds(30)); - } - } - finally - { - cts.Cancel(); - logTask.Wait(); - } - } - private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { ScopedTestFiles(() => { - var testFile = GenerateTestFile(10.MB()); + var testFile = GenerateTestFile(1000.MB()); var contentId = primary.UploadFile(testFile); From 598dc766fab7c802c8c95e0b011b5cd786cd9c7e Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 8 Aug 2023 09:44:38 +0200 Subject: [PATCH 03/31] use extra log codex image --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 4 ++-- Tests/BasicTests/ContinuousSubstitute.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ea6f187..08bad80 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,7 +5,7 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-7efa917"; + private const string DefaultDockerImage = "thatbenbierens/nim-codex:updownload"; public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index b634ff5..572cb9f 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -67,7 +67,7 @@ namespace DistTestCore { using var fileStream = File.OpenRead(file.Filename); - var logMessage = $"{CodexAccess.Container.Name} Uploading file {file.Describe()}..."; + var logMessage = $"Uploading file {file.Describe()}..."; Log(logMessage); var response = Stopwatch.Measure(lifecycle.Log, logMessage, () => { @@ -83,7 +83,7 @@ namespace DistTestCore public TestFile? DownloadContent(ContentId contentId, string fileLabel = "") { - var logMessage = $"{CodexAccess.Container.Name} Downloading for contentId: '{contentId.Id}'..."; + var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; Log(logMessage); var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel); Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file)); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index e589df0..324b3d7 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -16,7 +16,7 @@ namespace Tests.BasicTests { nodes.Add((OnlineCodexNode)SetupCodexNode(o => o .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) - .WithStorageQuota(2.GB()) + .WithStorageQuota(3.GB()) )); } From eaf5db5e91be356627161747d5ae7d2c9be94825 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 8 Aug 2023 10:45:16 +0200 Subject: [PATCH 04/31] Run test against increasing file sizes --- DistTestCore/ByteSize.cs | 10 ++++++++++ Tests/BasicTests/ContinuousSubstitute.cs | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs index 8d47cf2..bbf01cc 100644 --- a/DistTestCore/ByteSize.cs +++ b/DistTestCore/ByteSize.cs @@ -38,6 +38,11 @@ namespace DistTestCore { private const long Kilo = 1024; + public static ByteSize Bytes(this long i) + { + return new ByteSize(i); + } + public static ByteSize KB(this long i) { return new ByteSize(i * Kilo); @@ -58,6 +63,11 @@ namespace DistTestCore return (i * Kilo).GB(); } + public static ByteSize Bytes(this int i) + { + return new ByteSize(i); + } + public static ByteSize KB(this int i) { return Convert.ToInt64(i).KB(); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 324b3d7..ff606ef 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -61,17 +61,21 @@ namespace Tests.BasicTests } } + private ByteSize fileSize = 10.MB(); + private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { ScopedTestFiles(() => { - var testFile = GenerateTestFile(1000.MB()); + var testFile = GenerateTestFile(fileSize); var contentId = primary.UploadFile(testFile); var downloadedFile = secondary.DownloadContent(contentId); testFile.AssertIsEqual(downloadedFile); + + fileSize = (fileSize.SizeInBytes * 2).Bytes(); }); } } From 68d19353aadac4477d8ff95e8b6252ca5c734c9e Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 8 Aug 2023 14:42:59 +0200 Subject: [PATCH 05/31] moves to 80MB and long timeouts --- DistTestCore/Http.cs | 2 +- Tests/BasicTests/ContinuousSubstitute.cs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index b66ff75..ff52b42 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -76,7 +76,7 @@ namespace DistTestCore var content = new StreamContent(stream); content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); var response = Time.Wait(client.PostAsync(url, content)); - var str =Time.Wait(response.Content.ReadAsStringAsync()); + var str = Time.Wait(response.Content.ReadAsStringAsync()); Log(url, str); return str; }, $"HTTP-POST-STREAM: {route}"); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index ff606ef..40bfc77 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -9,6 +9,7 @@ namespace Tests.BasicTests public class ContinuousSubstitute : AutoBootstrapDistTest { [Test] + [UseLongTimeouts] public void ContinuousTestSubstitute() { var nodes = new List(); @@ -51,7 +52,7 @@ namespace Tests.BasicTests Log("Run Test"); PerformTest(primary, secondary); - Thread.Sleep(TimeSpan.FromSeconds(30)); + Thread.Sleep(TimeSpan.FromSeconds(5)); } } finally @@ -61,7 +62,7 @@ namespace Tests.BasicTests } } - private ByteSize fileSize = 10.MB(); + private ByteSize fileSize = 80.MB(); private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { @@ -74,8 +75,6 @@ namespace Tests.BasicTests var downloadedFile = secondary.DownloadContent(contentId); testFile.AssertIsEqual(downloadedFile); - - fileSize = (fileSize.SizeInBytes * 2).Bytes(); }); } } From 9037ddab6ea15fcb080094355146e316fdc34d5c Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 11 Aug 2023 09:37:30 +0200 Subject: [PATCH 06/31] initial setup of a grafana container --- .../Metrics/GrafanaContainerRecipe.cs | 19 +++++++++++++++++++ DistTestCore/PrometheusStarter.cs | 4 ++++ KubernetesWorkflow/ContainerRecipeFactory.cs | 7 +++++++ KubernetesWorkflow/RecipeComponentFactory.cs | 5 +++++ 4 files changed, 35 insertions(+) create mode 100644 DistTestCore/Metrics/GrafanaContainerRecipe.cs diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs new file mode 100644 index 0000000..5e52965 --- /dev/null +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -0,0 +1,19 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public class GrafanaContainerRecipe : ContainerRecipeFactory + { + public override string AppName => "grafana"; + public override string Image => "grafana/grafana-oss:10.0.3"; + + protected override void Initialize(StartupConfig startupConfig) + { + //var config = startupConfig.Get(); + + //AddExposedPortAndVar("PROM_PORT"); + AddExposedPort(3000); + //AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + } + } +} diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 1804b77..862907d 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -22,6 +22,10 @@ namespace DistTestCore var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + workflow = lifecycle.WorkflowCreator.CreateWorkflow(); + var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); + if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("should be 1"); + LogEnd("Metrics server started."); return runningContainers; diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 470bd42..36fff5b 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -40,6 +40,13 @@ return p; } + protected Port AddExposedPort(int number, string tag = "") + { + var p = factory.CreatePort(number, tag); + exposedPorts.Add(p); + return p; + } + protected Port AddInternalPort(string tag = "") { var p = factory.CreatePort(tag); diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index cf1f67b..9644515 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -7,6 +7,11 @@ namespace KubernetesWorkflow { private NumberSource portNumberSource = new NumberSource(8080); + public Port CreatePort(int number, string tag) + { + return new Port(number, tag); + } + public Port CreatePort(string tag) { return new Port(portNumberSource.GetNextNumber(), tag); From 79908b8a54768a073375957a03dad67a56f207fd Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 11 Aug 2023 10:16:19 +0200 Subject: [PATCH 07/31] disables grafana auth --- DistTestCore/Metrics/GrafanaContainerRecipe.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs index 5e52965..12af581 100644 --- a/DistTestCore/Metrics/GrafanaContainerRecipe.cs +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -14,6 +14,16 @@ namespace DistTestCore.Metrics //AddExposedPortAndVar("PROM_PORT"); AddExposedPort(3000); //AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + + // [auth.anonymous] + // enabled = true + //GF____FILE + + AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); + AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); + + //[auth] + //disable_login_form = true } } } From 2acb669c3120199ab7a1c8d425c99723a3ffbcfc Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 11 Aug 2023 11:06:40 +0200 Subject: [PATCH 08/31] automatically adds prometheus as datasource --- DistTestCore/Http.cs | 4 +- .../Metrics/GrafanaContainerRecipe.cs | 4 +- DistTestCore/PrometheusStarter.cs | 91 +++++++++++++++++++ 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 391dbc7..866a640 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,4 +1,5 @@ -using Logging; +using IdentityModel.Client; +using Logging; using Newtonsoft.Json; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -59,6 +60,7 @@ namespace DistTestCore var url = GetUrl() + route; using var content = JsonContent.Create(body); Log(url, JsonConvert.SerializeObject(body)); + client.SetBasicAuthentication("admin", "admin"); var result = Time.Wait(client.PostAsync(url, content)); var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs index 12af581..b873fa8 100644 --- a/DistTestCore/Metrics/GrafanaContainerRecipe.cs +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -19,8 +19,8 @@ namespace DistTestCore.Metrics // enabled = true //GF____FILE - AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); - AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); + //AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); + //AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); //[auth] //disable_login_form = true diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 862907d..4e27952 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -1,7 +1,14 @@ using DistTestCore.Codex; using DistTestCore.Metrics; using KubernetesWorkflow; +using Logging; +using System; +using System.Diagnostics; +using System.Net.Http.Headers; using System.Text; +using Utils; +using static System.Net.Mime.MediaTypeNames; +using static System.Net.WebRequestMethods; namespace DistTestCore { @@ -22,15 +29,99 @@ namespace DistTestCore var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + var pc = runningContainers.Containers.First().ClusterExternalAddress; + var prometheusUrl = pc.Host + ":" + pc.Port; + workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("should be 1"); + Thread.Sleep(3000); + + var c = grafanaContainers.Containers.First().ClusterExternalAddress; + + + + //{ + // //setup reusable http client + // HttpClient client = new HttpClient(); + // Uri baseUri = new Uri(c.Host + ":" + c.Port); + // client.BaseAddress = baseUri; + // client.DefaultRequestHeaders.Clear(); + // client.DefaultRequestHeaders.ConnectionClose = true; + + // //Post body content + // var values = new List>(); + // values.Add(new KeyValuePair("grant_type", "client_credentials")); + // var content = new FormUrlEncodedContent(values); + + // var authenticationString = $"admin:admin"; + // var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.ASCII.GetBytes(authenticationString)); + + // var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/oauth2/token"); + // requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString); + // requestMessage.Content = content; + + // //make the request + // var responsea = Time.Wait(client.SendAsync(requestMessage)); + // responsea.EnsureSuccessStatusCode(); + // string responseBody = Time.Wait(responsea.Content.ReadAsStringAsync()); + // Console.WriteLine(responseBody); + + //} + + //POST / api / datasources HTTP / 1.1 + //Accept: application / json + //Content - Type: application / json + //Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk + + var http = new Http(new NullLog(), new DefaultTimeSet(), c, "api/"); + var response = http.HttpPostJson("datasources", new GrafanaDataSource + { + name = "CodexPrometheus", + type = "prometheus", + url = prometheusUrl, + access = "proxy", + basicAuth = false, + jsonData = new GrafanaDataSourceJsonData + { + httpMethod = "POST" + } + }); + + + // [{ "id":1,"uid":"c89eaad3-9184-429f-ac94-8ba0b1824dbb","orgId":1, + // "name":"Prometheus","type":"prometheus","typeName":"Prometheus", + // "typeLogoUrl":"public/app/plugins/datasource/prometheus/img/prometheus_logo.svg", + // "access":"proxy","url":"http://kubernetes.docker.internal:31234","user":"","database":"", + // "basicAuth":false,"isDefault":true,"jsonData":{ "httpMethod":"POST"},"readOnly":false}] + + + var grafanaUrl = c.Host + ":" + c.Port; + System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); + LogEnd("Metrics server started."); + + return runningContainers; } + public class GrafanaDataSource + { + public string name { get; set; } = string.Empty; + public string type { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public string access { get; set; } = string.Empty; + public bool basicAuth { get; set; } + public GrafanaDataSourceJsonData jsonData { get; set; } = new(); + } + + public class GrafanaDataSourceJsonData + { + public string httpMethod { get; set; } = string.Empty; + } + private string GeneratePrometheusConfig(RunningContainer[] nodes) { var config = ""; From 485af03367c345096d316aae0059973585cb5dc0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 11 Aug 2023 12:38:26 +0200 Subject: [PATCH 09/31] very wip: automatic grafana dashboard --- DistTestCore/DistTestCore.csproj | 8 + DistTestCore/Http.cs | 17 +++ .../Metrics/GrafanaContainerRecipe.cs | 4 +- DistTestCore/Metrics/dashboard.json | 138 ++++++++++++++++++ DistTestCore/PrometheusStarter.cs | 68 +++------ 5 files changed, 183 insertions(+), 52 deletions(-) create mode 100644 DistTestCore/Metrics/dashboard.json diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 512d692..94a2271 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -10,6 +10,14 @@ Arm64 + + + + + + Never + + diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 866a640..1239fd7 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -68,6 +68,23 @@ namespace DistTestCore }, $"HTTP-POST-JSON: {route}"); } + public string HttpPostString(string route, string body) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + Log(url, body); + client.SetBasicAuthentication("admin", "admin"); + var content = new StringContent(body); + content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); + var result = Time.Wait(client.PostAsync(url, content)); + var str = Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; + }, $"HTTP-POST-STRING: {route}"); + } + public string HttpPostStream(string route, Stream stream) { return Retry(() => diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs index b873fa8..12af581 100644 --- a/DistTestCore/Metrics/GrafanaContainerRecipe.cs +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -19,8 +19,8 @@ namespace DistTestCore.Metrics // enabled = true //GF____FILE - //AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); - //AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); + AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); + AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); //[auth] //disable_login_form = true diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json new file mode 100644 index 0000000..59b6bbb --- /dev/null +++ b/DistTestCore/Metrics/dashboard.json @@ -0,0 +1,138 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexApiDownloads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Codex API Downloads", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Codex", + "uid": null, + "version": 2, + "weekStart": "" + }, + "message": "Default Codex Dashboard", + "overwrite": false +} diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 4e27952..d58ba6d 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -2,13 +2,8 @@ using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; -using System; -using System.Diagnostics; -using System.Net.Http.Headers; +using System.Reflection; using System.Text; -using Utils; -using static System.Net.Mime.MediaTypeNames; -using static System.Net.WebRequestMethods; namespace DistTestCore { @@ -40,44 +35,10 @@ namespace DistTestCore var c = grafanaContainers.Containers.First().ClusterExternalAddress; - - - //{ - // //setup reusable http client - // HttpClient client = new HttpClient(); - // Uri baseUri = new Uri(c.Host + ":" + c.Port); - // client.BaseAddress = baseUri; - // client.DefaultRequestHeaders.Clear(); - // client.DefaultRequestHeaders.ConnectionClose = true; - - // //Post body content - // var values = new List>(); - // values.Add(new KeyValuePair("grant_type", "client_credentials")); - // var content = new FormUrlEncodedContent(values); - - // var authenticationString = $"admin:admin"; - // var base64EncodedAuthenticationString = Convert.ToBase64String(System.Text.ASCIIEncoding.ASCII.GetBytes(authenticationString)); - - // var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/oauth2/token"); - // requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString); - // requestMessage.Content = content; - - // //make the request - // var responsea = Time.Wait(client.SendAsync(requestMessage)); - // responsea.EnsureSuccessStatusCode(); - // string responseBody = Time.Wait(responsea.Content.ReadAsStringAsync()); - // Console.WriteLine(responseBody); - - //} - - //POST / api / datasources HTTP / 1.1 - //Accept: application / json - //Content - Type: application / json - //Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk - var http = new Http(new NullLog(), new DefaultTimeSet(), c, "api/"); var response = http.HttpPostJson("datasources", new GrafanaDataSource { + uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", name = "CodexPrometheus", type = "prometheus", url = prometheusUrl, @@ -89,26 +50,19 @@ namespace DistTestCore } }); - - // [{ "id":1,"uid":"c89eaad3-9184-429f-ac94-8ba0b1824dbb","orgId":1, - // "name":"Prometheus","type":"prometheus","typeName":"Prometheus", - // "typeLogoUrl":"public/app/plugins/datasource/prometheus/img/prometheus_logo.svg", - // "access":"proxy","url":"http://kubernetes.docker.internal:31234","user":"","database":"", - // "basicAuth":false,"isDefault":true,"jsonData":{ "httpMethod":"POST"},"readOnly":false}] - + var response2 = http.HttpPostString("dashboards/db", GetDashboardJson()); var grafanaUrl = c.Host + ":" + c.Port; System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); LogEnd("Metrics server started."); - - return runningContainers; } public class GrafanaDataSource { + public string uid { get; set; } = string.Empty; public string name { get; set; } = string.Empty; public string type { get; set; } = string.Empty; public string url { get; set; } = string.Empty; @@ -122,6 +76,20 @@ namespace DistTestCore public string httpMethod { get; set; } = string.Empty; } + private string GetDashboardJson() + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = "DistTestCore.Metrics.dashboard.json"; + + //var names = assembly.GetManifestResourceNames(); + + using (Stream stream = assembly.GetManifestResourceStream(resourceName)) + using (StreamReader reader = new StreamReader(stream)) + { + return reader.ReadToEnd(); + } + } + private string GeneratePrometheusConfig(RunningContainer[] nodes) { var config = ""; From 75228c815dfb1576cf195cff9565c0e5b018e047 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 08:27:30 +0200 Subject: [PATCH 10/31] fixes dashboard permissions --- .../Metrics/GrafanaContainerRecipe.cs | 6 +- DistTestCore/Metrics/dashboard.json | 248 +++++++++--------- DistTestCore/PrometheusStarter.cs | 18 +- 3 files changed, 143 insertions(+), 129 deletions(-) diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs index 12af581..272001e 100644 --- a/DistTestCore/Metrics/GrafanaContainerRecipe.cs +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -20,7 +20,11 @@ namespace DistTestCore.Metrics //GF____FILE AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); - AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); + AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org."); + AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor"); + + //AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); + //AddEnvVar("GF_FEATURE_TOGGLES_ENABLE", "publicDashboards"); //[auth] //disable_login_form = true diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 59b6bbb..038a0f2 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -1,138 +1,134 @@ { - "dashboard": { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + "type": "grafana", + "uid": "-- Grafana --" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" - }, - "editorMode": "builder", - "expr": "codexApiDownloads_total", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Codex API Downloads", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Codex", - "uid": null, - "version": 2, - "weekStart": "" + "editorMode": "builder", + "expr": "codexApiDownloads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Codex API Downloads", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] }, - "message": "Default Codex Dashboard", - "overwrite": false -} + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Codex", + "uid": null, + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index d58ba6d..839061b 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -2,6 +2,7 @@ using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; +using Newtonsoft.Json; using System.Reflection; using System.Text; @@ -51,8 +52,9 @@ namespace DistTestCore }); var response2 = http.HttpPostString("dashboards/db", GetDashboardJson()); + var jsonResponse = JsonConvert.DeserializeObject(response2); - var grafanaUrl = c.Host + ":" + c.Port; + var grafanaUrl = c.Host + ":" + c.Port + jsonResponse.url; System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); LogEnd("Metrics server started."); @@ -76,6 +78,16 @@ namespace DistTestCore public string httpMethod { get; set; } = string.Empty; } + public class GrafanaPostDashboardResponse + { + public int id { get; set; } + public string slug { get; set; } = string.Empty; + public string status { get; set; } = string.Empty; + public string uid { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public int version { get; set; } + } + private string GetDashboardJson() { var assembly = Assembly.GetExecutingAssembly(); @@ -86,7 +98,9 @@ namespace DistTestCore using (Stream stream = assembly.GetManifestResourceStream(resourceName)) using (StreamReader reader = new StreamReader(stream)) { - return reader.ReadToEnd(); + var dashboard = reader.ReadToEnd(); + + return $"{{\"dashboard\": {dashboard} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; } } From fc3f424208b2c1cde9207cb353c2b22caf14c2f9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 08:40:32 +0200 Subject: [PATCH 11/31] Splits up grafana starter --- DistTestCore/GrafanaStarter.cs | 113 ++++++++++++++++++++++++++++++ DistTestCore/PrometheusStarter.cs | 79 --------------------- DistTestCore/TestLifecycle.cs | 2 + 3 files changed, 115 insertions(+), 79 deletions(-) create mode 100644 DistTestCore/GrafanaStarter.cs diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs new file mode 100644 index 0000000..b3ce38a --- /dev/null +++ b/DistTestCore/GrafanaStarter.cs @@ -0,0 +1,113 @@ +using DistTestCore.Metrics; +using KubernetesWorkflow; +using Logging; +using Newtonsoft.Json; +using System.Reflection; + +namespace DistTestCore +{ + public class GrafanaStarter : BaseStarter + { + public GrafanaStarter(TestLifecycle lifecycle) + : base(lifecycle) + { + } + public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer) + { + LogStart($"Starting dashboard server"); + var startupConfig = new StartupConfig(); + + var pc = prometheusContainer.ClusterExternalAddress; + var prometheusUrl = pc.Host + ":" + pc.Port; + + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); + var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); + if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created."); + + //Thread.Sleep(3000); + + var grafanaContainer = grafanaContainers.Containers.First(); + var c = grafanaContainer.ClusterExternalAddress; + + var http = new Http(new NullLog(), new DefaultTimeSet(), c, "api/"); + + // {"datasource":{"id":1,"uid":"c89eaad3-9184-429f-ac94-8ba0b1824dbb","orgId":1,"name":"CodexPrometheus","type":"prometheus","typeLogoUrl":"","access":"proxy","url":"http://kubernetes.docker.internal:31971","user":"","database":"","basicAuth":false,"basicAuthUser":"","withCredentials":false,"isDefault":false,"jsonData":{"httpMethod":"POST"},"secureJsonFields":{},"version":1,"readOnly":false},"id":1,"message":"Datasource added","name":"CodexPrometheus"} + var response = http.HttpPostJson("datasources", new GrafanaDataSource + { + uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", + name = "CodexPrometheus", + type = "prometheus", + url = prometheusUrl, + access = "proxy", + basicAuth = false, + jsonData = new GrafanaDataSourceJsonData + { + httpMethod = "POST" + } + }); + + var response2 = http.HttpPostString("dashboards/db", GetDashboardJson()); + var jsonResponse = JsonConvert.DeserializeObject(response2); + + var grafanaUrl = c.Host + ":" + c.Port + jsonResponse.url; + System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); + + LogEnd("Metrics server started."); + + return new GrafanaStartInfo(grafanaUrl, grafanaContainer); + } + + private string GetDashboardJson() + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = "DistTestCore.Metrics.dashboard.json"; + + using (Stream stream = assembly.GetManifestResourceStream(resourceName)) + using (StreamReader reader = new StreamReader(stream)) + { + var dashboard = reader.ReadToEnd(); + + return $"{{\"dashboard\": {dashboard} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; + } + } + } + + public class GrafanaStartInfo + { + public GrafanaStartInfo(string dashboardUrl, RunningContainer container) + { + DashboardUrl = dashboardUrl; + Container = container; + } + + public string DashboardUrl { get; } + public RunningContainer Container { get; } + } + + public class GrafanaDataSource + { + public string uid { get; set; } = string.Empty; + public string name { get; set; } = string.Empty; + public string type { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public string access { get; set; } = string.Empty; + public bool basicAuth { get; set; } + public GrafanaDataSourceJsonData jsonData { get; set; } = new(); + } + + public class GrafanaDataSourceJsonData + { + public string httpMethod { get; set; } = string.Empty; + } + + public class GrafanaPostDashboardResponse + { + public int id { get; set; } + public string slug { get; set; } = string.Empty; + public string status { get; set; } = string.Empty; + public string uid { get; set; } = string.Empty; + public string url { get; set; } = string.Empty; + public int version { get; set; } + } + +} diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 839061b..3b1e49c 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -1,9 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Metrics; using KubernetesWorkflow; -using Logging; -using Newtonsoft.Json; -using System.Reflection; using System.Text; namespace DistTestCore @@ -25,85 +22,9 @@ namespace DistTestCore var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); - var pc = runningContainers.Containers.First().ClusterExternalAddress; - var prometheusUrl = pc.Host + ":" + pc.Port; - - workflow = lifecycle.WorkflowCreator.CreateWorkflow(); - var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); - if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("should be 1"); - - Thread.Sleep(3000); - - var c = grafanaContainers.Containers.First().ClusterExternalAddress; - - var http = new Http(new NullLog(), new DefaultTimeSet(), c, "api/"); - var response = http.HttpPostJson("datasources", new GrafanaDataSource - { - uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", - name = "CodexPrometheus", - type = "prometheus", - url = prometheusUrl, - access = "proxy", - basicAuth = false, - jsonData = new GrafanaDataSourceJsonData - { - httpMethod = "POST" - } - }); - - var response2 = http.HttpPostString("dashboards/db", GetDashboardJson()); - var jsonResponse = JsonConvert.DeserializeObject(response2); - - var grafanaUrl = c.Host + ":" + c.Port + jsonResponse.url; - System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); - - LogEnd("Metrics server started."); - return runningContainers; } - public class GrafanaDataSource - { - public string uid { get; set; } = string.Empty; - public string name { get; set; } = string.Empty; - public string type { get; set; } = string.Empty; - public string url { get; set; } = string.Empty; - public string access { get; set; } = string.Empty; - public bool basicAuth { get; set; } - public GrafanaDataSourceJsonData jsonData { get; set; } = new(); - } - - public class GrafanaDataSourceJsonData - { - public string httpMethod { get; set; } = string.Empty; - } - - public class GrafanaPostDashboardResponse - { - public int id { get; set; } - public string slug { get; set; } = string.Empty; - public string status { get; set; } = string.Empty; - public string uid { get; set; } = string.Empty; - public string url { get; set; } = string.Empty; - public int version { get; set; } - } - - private string GetDashboardJson() - { - var assembly = Assembly.GetExecutingAssembly(); - var resourceName = "DistTestCore.Metrics.dashboard.json"; - - //var names = assembly.GetManifestResourceNames(); - - using (Stream stream = assembly.GetManifestResourceStream(resourceName)) - using (StreamReader reader = new StreamReader(stream)) - { - var dashboard = reader.ReadToEnd(); - - return $"{{\"dashboard\": {dashboard} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; - } - } - private string GeneratePrometheusConfig(RunningContainer[] nodes) { var config = ""; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index eede32d..0958898 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -24,6 +24,7 @@ namespace DistTestCore FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this); PrometheusStarter = new PrometheusStarter(this); + GrafanaStarter = new GrafanaStarter(this); GethStarter = new GethStarter(this); testStart = DateTime.UtcNow; CodexVersion = null; @@ -38,6 +39,7 @@ namespace DistTestCore public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } + public GrafanaStarter GrafanaStarter { get; } public GethStarter GethStarter { get; } public CodexDebugVersionResponse? CodexVersion { get; private set; } From f1131abf793aabd70d9924877c391c5659dffd1e Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 09:07:23 +0200 Subject: [PATCH 12/31] Wires dashboard into deployer --- CodexNetDeployer/CodexNodeStarter.cs | 2 +- CodexNetDeployer/Configuration.cs | 5 +++-- CodexNetDeployer/Deployer.cs | 18 ++++++++++++------ CodexNetDeployer/deploy-continuous-testnet.sh | 2 +- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- DistTestCore/Codex/CodexDeployment.cs | 4 +++- DistTestCore/Codex/CodexStartupConfig.cs | 3 ++- DistTestCore/CodexSetup.cs | 2 +- DistTestCore/CodexStarter.cs | 8 +++++++- DistTestCore/Metrics/MetricsMode.cs | 9 +++++++++ 10 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 DistTestCore/Metrics/MetricsMode.cs diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 0a3b373..836e8f8 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -84,7 +84,7 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; - codexStart.MetricsEnabled = config.RecordMetrics; + codexStart.MetricsMode = config.Metrics; if (config.BlockTTL != Configuration.SecondsIn1Day) { diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index ee7ac7e..0257389 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,5 +1,6 @@ using ArgsUniform; using DistTestCore.Codex; +using DistTestCore.Metrics; namespace CodexNetDeployer { @@ -43,8 +44,8 @@ namespace CodexNetDeployer [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")] public int BlockTTL { get; set; } = SecondsIn1Day; - [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] - public bool RecordMetrics { get; set; } = false; + [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")] + public MetricsMode Metrics { get; set; } = MetricsMode.None; [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + "set this option to override the label value.")] diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index feb750e..e790320 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -27,7 +27,7 @@ namespace CodexNetDeployer // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. var setup = new CodexSetup(999, config.CodexLogLevel); setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); - setup.MetricsEnabled = config.RecordMetrics; + setup.MetricsMode = config.Metrics; Log("Creating Geth instance and deploying contracts..."); var gethStarter = new GethStarter(lifecycle); @@ -52,9 +52,9 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers); + var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers); - return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); + return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } private TestLifecycle CreateTestLifecycle() @@ -74,13 +74,19 @@ namespace CodexNetDeployer return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } - private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) { - if (!setup.MetricsEnabled) return null; + if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null); Log("Starting metrics service..."); var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) }; - return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); + var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); + + if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null); + + Log("Starting dashboard service..."); + var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer); + return (prometheusContainer, grafanaStartInfo); } private string? GetKubeConfig(string kubeConfigFile) diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 338b8c1..c0460c8 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -10,4 +10,4 @@ dotnet run \ --max-collateral=1024 \ --max-duration=3600000 \ --block-ttl=300 \ - --record-metrics=true + --metrics=Dashboard diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index c600c01..64f85c3 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -51,7 +51,7 @@ namespace DistTestCore.Codex { AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!); } - if (config.MetricsEnabled) + if (config.MetricsMode != Metrics.MetricsMode.None) { AddEnvVar("CODEX_METRICS", "true"); AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 52192dc..b22c77a 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,17 +5,19 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; PrometheusContainer = prometheusContainer; + GrafanaStartInfo = grafanaStartInfo; Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } public RunningContainer? PrometheusContainer { get; } + public GrafanaStartInfo? GrafanaStartInfo { get; } public DeploymentMetadata Metadata { get; } } diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 2a17334..5e72755 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,5 @@ using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; namespace DistTestCore.Codex @@ -14,7 +15,7 @@ namespace DistTestCore.Codex public Location Location { get; set; } public CodexLogLevel LogLevel { get; } public ByteSize? StorageQuota { get; set; } - public bool MetricsEnabled { get; set; } + public MetricsMode MetricsMode { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public string? BootstrapSpr { get; set; } public int? BlockTTL { get; set; } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 0ffb713..4aefd39 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -59,7 +59,7 @@ namespace DistTestCore public ICodexSetup EnableMetrics() { - MetricsEnabled = true; + MetricsMode = Metrics.MetricsMode.Record; return this; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index a1e38b7..52686f1 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -68,9 +68,15 @@ namespace DistTestCore private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers) { - if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory(); var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers); + + if (codexSetup.MetricsMode == MetricsMode.Dashboard) + { + var info = lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); + } + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); } diff --git a/DistTestCore/Metrics/MetricsMode.cs b/DistTestCore/Metrics/MetricsMode.cs new file mode 100644 index 0000000..60b4f5e --- /dev/null +++ b/DistTestCore/Metrics/MetricsMode.cs @@ -0,0 +1,9 @@ +namespace DistTestCore.Metrics +{ + public enum MetricsMode + { + None, + Record, + Dashboard + } +} From de662363833d1e1781beb08213b09943d7881ebd Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 10:01:06 +0200 Subject: [PATCH 13/31] Cleans up grafana starter. --- DistTestCore/CodexStarter.cs | 2 +- DistTestCore/GrafanaStarter.cs | 105 ++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 52686f1..3c35e42 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -74,7 +74,7 @@ namespace DistTestCore if (codexSetup.MetricsMode == MetricsMode.Dashboard) { - var info = lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); + lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); } return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index b3ce38a..82a4614 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -1,8 +1,8 @@ using DistTestCore.Metrics; using KubernetesWorkflow; -using Logging; using Newtonsoft.Json; using System.Reflection; +using Utils; namespace DistTestCore { @@ -15,24 +15,40 @@ namespace DistTestCore public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer) { LogStart($"Starting dashboard server"); + + var grafanaContainer = StartGrafanaContainer(); + var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer); + + var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/"); + + Log("Connecting datasource..."); + AddDataSource(http, prometheusContainer); + + Log("Uploading dashboard configurations..."); + var jsons = ReadEachDashboardJsonFile(); + var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaAddress, j)).ToArray(); + + LogEnd("Dashboard server started."); + + return new GrafanaStartInfo(dashboardUrls, grafanaContainer); + } + + private RunningContainer StartGrafanaContainer() + { var startupConfig = new StartupConfig(); - - var pc = prometheusContainer.ClusterExternalAddress; - var prometheusUrl = pc.Host + ":" + pc.Port; var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig); if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created."); - //Thread.Sleep(3000); + return grafanaContainers.Containers.First(); + } - var grafanaContainer = grafanaContainers.Containers.First(); - var c = grafanaContainer.ClusterExternalAddress; - - var http = new Http(new NullLog(), new DefaultTimeSet(), c, "api/"); - - // {"datasource":{"id":1,"uid":"c89eaad3-9184-429f-ac94-8ba0b1824dbb","orgId":1,"name":"CodexPrometheus","type":"prometheus","typeLogoUrl":"","access":"proxy","url":"http://kubernetes.docker.internal:31971","user":"","database":"","basicAuth":false,"basicAuthUser":"","withCredentials":false,"isDefault":false,"jsonData":{"httpMethod":"POST"},"secureJsonFields":{},"version":1,"readOnly":false},"id":1,"message":"Datasource added","name":"CodexPrometheus"} - var response = http.HttpPostJson("datasources", new GrafanaDataSource + private static void AddDataSource(Http http, RunningContainer prometheusContainer) + { + var prometheusAddress = prometheusContainer.ClusterExternalAddress; + var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port; + var response = http.HttpPostJson("datasources", new GrafanaDataSourceRequest { uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb", name = "CodexPrometheus", @@ -46,45 +62,60 @@ namespace DistTestCore } }); - var response2 = http.HttpPostString("dashboards/db", GetDashboardJson()); - var jsonResponse = JsonConvert.DeserializeObject(response2); - - var grafanaUrl = c.Host + ":" + c.Port + jsonResponse.url; - System.Diagnostics.Process.Start("C:\\Users\\Ben\\AppData\\Local\\Programs\\Opera\\opera.exe", grafanaUrl); - - LogEnd("Metrics server started."); - - return new GrafanaStartInfo(grafanaUrl, grafanaContainer); + if (response.message != "Datasource added") + { + throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message); + } } - private string GetDashboardJson() + public static string UploadDashboard(Http http, Address grafanaAddress, string dashboardJson) + { + var request = GetDashboardCreateRequest(dashboardJson); + var response = http.HttpPostString("dashboards/db", request); + var jsonResponse = JsonConvert.DeserializeObject(response); + if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard."); + + return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; + } + + private static string[] ReadEachDashboardJsonFile() { var assembly = Assembly.GetExecutingAssembly(); - var resourceName = "DistTestCore.Metrics.dashboard.json"; - - using (Stream stream = assembly.GetManifestResourceStream(resourceName)) - using (StreamReader reader = new StreamReader(stream)) + var resourceNames = new[] { - var dashboard = reader.ReadToEnd(); + "DistTestCore.Metrics.dashboard.json" + }; - return $"{{\"dashboard\": {dashboard} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; - } + return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray(); + } + + private static string GetManifestResource(Assembly assembly, string resourceName) + { + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) throw new Exception("Unable to find resource " + resourceName); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + private static string GetDashboardCreateRequest(string dashboardJson) + { + return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}"; } } public class GrafanaStartInfo { - public GrafanaStartInfo(string dashboardUrl, RunningContainer container) + public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container) { - DashboardUrl = dashboardUrl; + DashboardUrls = dashboardUrls; Container = container; } - public string DashboardUrl { get; } + public string[] DashboardUrls { get; } public RunningContainer Container { get; } } - public class GrafanaDataSource + public class GrafanaDataSourceRequest { public string uid { get; set; } = string.Empty; public string name { get; set; } = string.Empty; @@ -95,6 +126,13 @@ namespace DistTestCore public GrafanaDataSourceJsonData jsonData { get; set; } = new(); } + public class GrafanaDataSourceResponse + { + public int id { get; set; } + public string message { get; set; } = string.Empty; + public string name { get; set; } = string.Empty; + } + public class GrafanaDataSourceJsonData { public string httpMethod { get; set; } = string.Empty; @@ -109,5 +147,4 @@ namespace DistTestCore public string url { get; set; } = string.Empty; public int version { get; set; } } - } From bce8a48cade7a915e00381d869beaf6eece4794f Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 10:07:47 +0200 Subject: [PATCH 14/31] logs grafana image id --- CodexNetDeployer/Program.cs | 3 ++- DistTestCore/TestLifecycle.cs | 3 ++- KubernetesWorkflow/PodLabels.cs | 1 + Logging/ApplicationIds.cs | 4 +++- Logging/StatusLog.cs | 2 ++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 9c1ecbf..da18e01 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -30,7 +30,8 @@ public class Program $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl + $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl + $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl + - $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl); + $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl + + $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl); if (!args.Any(a => a == "-y")) { diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 0958898..260e438 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -78,7 +78,8 @@ namespace DistTestCore codexId: GetCodexId(), gethId: new GethContainerRecipe().Image, prometheusId: new PrometheusContainerRecipe().Image, - codexContractsId: new CodexContractsContainerRecipe().Image + codexContractsId: new CodexContractsContainerRecipe().Image, + grafanaId: new GrafanaContainerRecipe().Image ); } diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs index 60fc332..85b3a98 100644 --- a/KubernetesWorkflow/PodLabels.cs +++ b/KubernetesWorkflow/PodLabels.cs @@ -25,6 +25,7 @@ namespace KubernetesWorkflow Add("gethid", applicationIds.GethId); Add("prometheusid", applicationIds.PrometheusId); Add("codexcontractsid", applicationIds.CodexContractsId); + Add("grafanaid", applicationIds.GrafanaId); } public PodLabels GetLabelsForAppName(string appName) diff --git a/Logging/ApplicationIds.cs b/Logging/ApplicationIds.cs index d52dc10..7162a36 100644 --- a/Logging/ApplicationIds.cs +++ b/Logging/ApplicationIds.cs @@ -2,17 +2,19 @@ { public class ApplicationIds { - public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId) + public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId) { CodexId = codexId; GethId = gethId; PrometheusId = prometheusId; CodexContractsId = codexContractsId; + GrafanaId = grafanaId; } public string CodexId { get; } public string GethId { get; } public string PrometheusId { get; } public string CodexContractsId { get; } + public string GrafanaId { get; } } } diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs index 96a8cc2..f5d5831 100644 --- a/Logging/StatusLog.cs +++ b/Logging/StatusLog.cs @@ -26,6 +26,7 @@ namespace Logging gethid = applicationIds.GethId, prometheusid = applicationIds.PrometheusId, codexcontractsid = applicationIds.CodexContractsId, + grafanaid = applicationIds.GrafanaId, category = NameUtils.GetCategoryName(), fixturename = fixtureName, testname = NameUtils.GetTestMethodName(), @@ -59,6 +60,7 @@ namespace Logging public string gethid { get; set; } = string.Empty; public string prometheusid { get; set; } = string.Empty; public string codexcontractsid { get; set; } = string.Empty; + public string grafanaid { get; set; } = string.Empty; public string category { get; set; } = string.Empty; public string fixturename { get; set; } = string.Empty; public string testname { get; set; } = string.Empty; From 75cdb94e274078cb3274ae52e0a780e92a8ef3e6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 10:10:47 +0200 Subject: [PATCH 15/31] use cluster-external addresses for dashboard URLs --- DistTestCore/GrafanaStarter.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 82a4614..f66db8c 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -26,7 +26,7 @@ namespace DistTestCore Log("Uploading dashboard configurations..."); var jsons = ReadEachDashboardJsonFile(); - var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaAddress, j)).ToArray(); + var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray(); LogEnd("Dashboard server started."); @@ -68,13 +68,14 @@ namespace DistTestCore } } - public static string UploadDashboard(Http http, Address grafanaAddress, string dashboardJson) + public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson) { var request = GetDashboardCreateRequest(dashboardJson); var response = http.HttpPostString("dashboards/db", request); var jsonResponse = JsonConvert.DeserializeObject(response); if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard."); + var grafanaAddress = grafanaContainer.ClusterExternalAddress; return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; } From b4302ab6f76b1263f8c26af3d066f99647b17f7c Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 13 Aug 2023 11:19:35 +0200 Subject: [PATCH 16/31] Decent dashboard setup --- DistTestCore/GrafanaStarter.cs | 11 +- DistTestCore/Http.cs | 17 +- .../Metrics/GrafanaContainerRecipe.cs | 18 +- DistTestCore/Metrics/dashboard.json | 1965 +++++++++++++++-- 4 files changed, 1864 insertions(+), 147 deletions(-) diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index f66db8c..61bb17d 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -1,8 +1,8 @@ using DistTestCore.Metrics; +using IdentityModel.Client; using KubernetesWorkflow; using Newtonsoft.Json; using System.Reflection; -using Utils; namespace DistTestCore { @@ -19,7 +19,7 @@ namespace DistTestCore var grafanaContainer = StartGrafanaContainer(); var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer); - var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/"); + var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth); Log("Connecting datasource..."); AddDataSource(http, prometheusContainer); @@ -44,6 +44,13 @@ namespace DistTestCore return grafanaContainers.Containers.First(); } + private void AddBasicAuth(HttpClient client) + { + client.SetBasicAuthentication( + GrafanaContainerRecipe.DefaultAdminUser, + GrafanaContainerRecipe.DefaultAdminPassword); + } + private static void AddDataSource(Http http, RunningContainer prometheusContainer) { var prometheusAddress = prometheusContainer.ClusterExternalAddress; diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 1239fd7..0d99db0 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,5 +1,4 @@ -using IdentityModel.Client; -using Logging; +using Logging; using Newtonsoft.Json; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -13,14 +12,21 @@ namespace DistTestCore private readonly ITimeSet timeSet; private readonly Address address; private readonly string baseUrl; + private readonly Action onClientCreated; private readonly string? logAlias; public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null) + : this(log, timeSet, address, baseUrl, DoNothing, logAlias) + { + } + + public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action onClientCreated, string? logAlias = null) { this.log = log; this.timeSet = timeSet; this.address = address; this.baseUrl = baseUrl; + this.onClientCreated = onClientCreated; this.logAlias = logAlias; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; @@ -60,7 +66,6 @@ namespace DistTestCore var url = GetUrl() + route; using var content = JsonContent.Create(body); Log(url, JsonConvert.SerializeObject(body)); - client.SetBasicAuthentication("admin", "admin"); var result = Time.Wait(client.PostAsync(url, content)); var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); @@ -75,7 +80,6 @@ namespace DistTestCore using var client = GetClient(); var url = GetUrl() + route; Log(url, body); - client.SetBasicAuthentication("admin", "admin"); var content = new StringContent(body); content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); var result = Time.Wait(client.PostAsync(url, content)); @@ -151,7 +155,12 @@ namespace DistTestCore { var client = new HttpClient(); client.Timeout = timeSet.HttpCallTimeout(); + onClientCreated(client); return client; } + + private static void DoNothing(HttpClient client) + { + } } } diff --git a/DistTestCore/Metrics/GrafanaContainerRecipe.cs b/DistTestCore/Metrics/GrafanaContainerRecipe.cs index 272001e..455af39 100644 --- a/DistTestCore/Metrics/GrafanaContainerRecipe.cs +++ b/DistTestCore/Metrics/GrafanaContainerRecipe.cs @@ -7,27 +7,19 @@ namespace DistTestCore.Metrics public override string AppName => "grafana"; public override string Image => "grafana/grafana-oss:10.0.3"; + public const string DefaultAdminUser = "adminium"; + public const string DefaultAdminPassword = "passwordium"; + protected override void Initialize(StartupConfig startupConfig) { - //var config = startupConfig.Get(); - - //AddExposedPortAndVar("PROM_PORT"); AddExposedPort(3000); - //AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); - - // [auth.anonymous] - // enabled = true - //GF____FILE AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true"); AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org."); AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor"); - //AddEnvVar("GF_AUTH_DISABLE_LOGIN_FORM", "true"); - //AddEnvVar("GF_FEATURE_TOGGLES_ENABLE", "publicDashboards"); - - //[auth] - //disable_login_form = true + AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser); + AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword); } } } diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 038a0f2..591f0c3 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -1,134 +1,1843 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "API Upload/Download", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" - }, - "editorMode": "builder", - "expr": "codexApiDownloads_total", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Codex API Downloads", - "type": "timeseries" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "refresh": "10s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexApiUploads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Uploads", + "type": "timeseries" }, - "time": { - "from": "now-1h", - "to": "now" + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexApiDownloads_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Downloads", + "type": "timeseries" }, - "timepicker": {}, - "timezone": "", - "title": "Codex", - "uid": null, - "version": 2, - "weekStart": "" + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "title": "Local Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBlocks", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Number of Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBytesReserved", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Bytes reserved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexRepostoreBytesUsed", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Bytes used", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 8, + "panels": [], + "title": "Block Exchange", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeBlocksSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Blocks sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeBlocksReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Blocks received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantHaveListsSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantBlockListsSent_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantHaveListsReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantHave lists received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexBlockExchangeWantBlockListsReceived_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "WantBlock lists received", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "panels": [], + "title": "Purchases", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesPending_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesSubmitted_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Submitted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesStarted_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Started", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesFinished_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Finished", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 44 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesFailed_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Failed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 44 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesCancelled_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Cancelled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 44 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesUnknown_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Unknown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 44 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "codexPurchasesError_total", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Error", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Codex", + "uid": "ca93f344-dd41-48c4-95a7-876c340e3005", + "version": 6, + "weekStart": "" } \ No newline at end of file From 9f0600d6c190a9085719f694eeba63b3e02e25f5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 14 Aug 2023 10:26:04 +0200 Subject: [PATCH 17/31] Automatic quota line. Holding test. --- CodexNetDeployer/Deployer.cs | 2 +- DistTestCore/CodexStarter.cs | 2 +- DistTestCore/GrafanaStarter.cs | 45 ++++++++-- DistTestCore/Http.cs | 9 +- DistTestCore/Metrics/dashboard.json | 6 +- Tests/BasicTests/ContinuousSubstitute.cs | 103 +++++++++++++---------- 6 files changed, 106 insertions(+), 61 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index e790320..859defd 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -85,7 +85,7 @@ namespace CodexNetDeployer if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null); Log("Starting dashboard service..."); - var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer); + var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup); return (prometheusContainer, grafanaStartInfo); } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 3c35e42..14278c7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -74,7 +74,7 @@ namespace DistTestCore if (codexSetup.MetricsMode == MetricsMode.Dashboard) { - lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First()); + lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup); } return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 61bb17d..679122d 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -8,11 +8,14 @@ namespace DistTestCore { public class GrafanaStarter : BaseStarter { + private const string StorageQuotaThresholdReplaceToken = "\"\""; + private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"\""; + public GrafanaStarter(TestLifecycle lifecycle) : base(lifecycle) { } - public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer) + public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup) { LogStart($"Starting dashboard server"); @@ -25,7 +28,7 @@ namespace DistTestCore AddDataSource(http, prometheusContainer); Log("Uploading dashboard configurations..."); - var jsons = ReadEachDashboardJsonFile(); + var jsons = ReadEachDashboardJsonFile(codexSetup); var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray(); LogEnd("Dashboard server started."); @@ -86,7 +89,7 @@ namespace DistTestCore return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url; } - private static string[] ReadEachDashboardJsonFile() + private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup) { var assembly = Assembly.GetExecutingAssembly(); var resourceNames = new[] @@ -94,15 +97,45 @@ namespace DistTestCore "DistTestCore.Metrics.dashboard.json" }; - return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray(); + return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray(); } - private static string GetManifestResource(Assembly assembly, string resourceName) + private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup) { using var stream = assembly.GetManifestResourceStream(resourceName); if (stream == null) throw new Exception("Unable to find resource " + resourceName); using var reader = new StreamReader(stream); - return reader.ReadToEnd(); + return ApplyReplacements(reader.ReadToEnd(), codexSetup); + } + + private static string ApplyReplacements(string input, CodexSetup codexSetup) + { + var quotaString = GetQuotaString(codexSetup); + var softMaxString = GetSoftMaxString(codexSetup); + + return input + .Replace(StorageQuotaThresholdReplaceToken, quotaString) + .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString); + } + + private static string GetQuotaString(CodexSetup codexSetup) + { + return GetCodexStorageQuotaInBytes(codexSetup).ToString(); + } + + private static string GetSoftMaxString(CodexSetup codexSetup) + { + var quota = GetCodexStorageQuotaInBytes(codexSetup); + var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing. + return softMax.ToString(); + } + + private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup) + { + if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes; + + // Codex default: 8GB + return 8.GB().SizeInBytes; } private static string GetDashboardCreateRequest(string dashboardJson) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index b14df70..c2c0709 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -148,14 +148,7 @@ namespace DistTestCore private T Retry(Func operation, string description) { - try - { - return operation(); - } - catch (Exception ex) - { - throw new Exception(description, ex); - } + return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 591f0c3..d53d72b 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -431,6 +431,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": "", + "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -453,7 +455,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "line" } }, "mappings": [], @@ -466,7 +468,7 @@ }, { "color": "red", - "value": 80 + "value": "" } ] } diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 40bfc77..5cc010a 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,5 +1,4 @@ -using ContinuousTests; -using DistTestCore; +using DistTestCore; using NUnit.Framework; using Utils; @@ -12,53 +11,34 @@ namespace Tests.BasicTests [UseLongTimeouts] public void ContinuousTestSubstitute() { - var nodes = new List(); - for (var i = 0; i < 5; i++) + var group = SetupCodexNodes(5, o => o + .EnableMetrics() + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(3.GB())); + + var nodes = group.Cast().ToArray(); + + foreach (var node in nodes) { - nodes.Add((OnlineCodexNode)SetupCodexNode(o => o - .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2) - .WithStorageQuota(3.GB()) - )); + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); } - var cts = new CancellationTokenSource(); - var ct = cts.Token; - var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs"); - Directory.CreateDirectory(dlPath); - - var containers = nodes.Select(n => n.CodexAccess.Container).ToArray(); - var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct); - - var logTask = Task.Run(cd.Run); - - try + var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); + while (DateTime.UtcNow < endTime) { - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } + var allNodes = nodes.ToList(); + var primary = allNodes.PickOneRandom(); + var secondary = allNodes.PickOneRandom(); - var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); - while (DateTime.UtcNow < endTime) - { - var allNodes = nodes.ToList(); - var primary = allNodes.PickOneRandom(); - var secondary = allNodes.PickOneRandom(); + Log("Run Test"); + PerformTest(primary, secondary); - Log("Run Test"); - PerformTest(primary, secondary); - - Thread.Sleep(TimeSpan.FromSeconds(5)); - } - } - finally - { - cts.Cancel(); - logTask.Wait(); + Thread.Sleep(TimeSpan.FromSeconds(5)); } } @@ -77,5 +57,42 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); }); } + + [Test] + [UseLongTimeouts] + public void HoldMyBeerTest() + { + var group = SetupCodexNodes(5, o => o + .EnableMetrics() + .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(3.GB())); + + var nodes = group.Cast().ToArray(); + + foreach (var node in nodes) + { + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); + } + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + while (DateTime.UtcNow < endTime) + { + foreach (var node in nodes) + { + var file = GenerateTestFile(80.MB()); + var cid = node.UploadFile(file); + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + } + + Thread.Sleep(TimeSpan.FromMinutes(2)); + } + } } } From 6e5e30afd429236ae64603be01221bc2f656b64e Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 14 Aug 2023 15:10:36 +0200 Subject: [PATCH 18/31] rigged up a pod crash detector maybe --- DistTestCore/Metrics/dashboard.json | 6 ++- KubernetesWorkflow/K8sController.cs | 24 +++++++++ KubernetesWorkflow/StartupWorkflow.cs | 5 ++ Tests/BasicTests/ContinuousSubstitute.cs | 62 +++++++++++++++++------- 4 files changed, 77 insertions(+), 20 deletions(-) diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index d53d72b..fdc5a2a 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -377,7 +377,8 @@ "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, @@ -471,7 +472,8 @@ "value": "" } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 5569b07..29147ea 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -604,6 +604,30 @@ namespace KubernetesWorkflow #endregion + public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler) + { + return Task.Run(() => + { + var myOwnClient = new Kubernetes(cluster.GetK8sClientConfig()); + while (!token.IsCancellationRequested) + { + token.WaitHandle.WaitOne(TimeSpan.FromSeconds(3)); + + var pod = container.Pod; + var recipe = container.Recipe; + var podName = pod.PodInfo.Name; + var podInfo = myOwnClient.ReadNamespacedPod(podName, K8sTestNamespace); + + if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) + { + log.Log("Pod crash detected for " + container.Name); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true)); + logHandler.Log(stream); + } + } + }); + } + private PodInfo FetchNewPod() { var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c51b991..6f390a0 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -37,6 +37,11 @@ namespace KubernetesWorkflow }, pl); } + public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler) + { + return K8s(controller => controller.WatchForCrashLogs(container, token, logHandler)); + } + public void Stop(RunningContainers runningContainers) { K8s(controller => diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 5cc010a..722bb3f 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,11 +1,12 @@ using DistTestCore; +using KubernetesWorkflow; using NUnit.Framework; using Utils; namespace Tests.BasicTests { [TestFixture] - public class ContinuousSubstitute : AutoBootstrapDistTest + public class ContinuousSubstitute : AutoBootstrapDistTest, ILogHandler { [Test] [UseLongTimeouts] @@ -57,7 +58,7 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); }); } - + [Test] [UseLongTimeouts] public void HoldMyBeerTest() @@ -70,28 +71,53 @@ namespace Tests.BasicTests var nodes = group.Cast().ToArray(); - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } + var flow = Get().WorkflowCreator.CreateWorkflow(); + var cst = new CancellationTokenSource(); + var tasks = nodes.Select(n => flow.WatchForCrashLogs(n.CodexAccess.Container, cst.Token, this)).ToArray(); - var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); - while (DateTime.UtcNow < endTime) + try { foreach (var node in nodes) { - var file = GenerateTestFile(80.MB()); - var cid = node.UploadFile(file); - - var dl = node.DownloadContent(cid); - file.AssertIsEqual(dl); + node.Marketplace.MakeStorageAvailable( + size: 1.GB(), + minPricePerBytePerSecond: 1.TestTokens(), + maxCollateral: 1024.TestTokens(), + maxDuration: TimeSpan.FromMinutes(5)); } - Thread.Sleep(TimeSpan.FromMinutes(2)); + var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + while (DateTime.UtcNow < endTime) + { + foreach (var node in nodes) + { + var file = GenerateTestFile(80.MB()); + var cid = node.UploadFile(file); + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + } + + Thread.Sleep(TimeSpan.FromMinutes(2)); + } + } + finally + { + cst.Cancel(); + foreach (var t in tasks) t.Wait(); + } + } + + public void Log(Stream log) + { + Log("Well damn, container crashed. Here's the log:"); + using var reader = new StreamReader(log); + + var line = reader.ReadLine(); + while(line != null) + { + Log(line); + line = reader.ReadLine(); } } } From c0b7d3a7477d9c0479da20d5a6b8cace58763985 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 14 Aug 2023 15:51:03 +0200 Subject: [PATCH 19/31] Hooks up block maintenance interval config param. --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 ++++ DistTestCore/Codex/CodexStartupConfig.cs | 1 + DistTestCore/CodexSetup.cs | 7 +++++++ DistTestCore/GrafanaStarter.cs | 1 + Tests/BasicTests/ContinuousSubstitute.cs | 1 + 5 files changed, 14 insertions(+) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 2f1f383..0a35afe 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -51,6 +51,10 @@ namespace DistTestCore.Codex { AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!); } + if (config.BlockMaintenanceInterval != null) + { + AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString()); + } if (config.MetricsMode != Metrics.MetricsMode.None) { AddEnvVar("CODEX_METRICS", "true"); diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 5e72755..3dc9218 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -19,5 +19,6 @@ namespace DistTestCore.Codex public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public string? BootstrapSpr { get; set; } public int? BlockTTL { get; set; } + public TimeSpan? BlockMaintenanceInterval { get; set; } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 4aefd39..1eb99d7 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -11,6 +11,7 @@ namespace DistTestCore ICodexSetup WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup WithBlockTTL(TimeSpan duration); + ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); @@ -57,6 +58,12 @@ namespace DistTestCore return this; } + public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration) + { + BlockMaintenanceInterval = duration; + return this; + } + public ICodexSetup EnableMetrics() { MetricsMode = Metrics.MetricsMode.Record; diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 679122d..978c0f4 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -15,6 +15,7 @@ namespace DistTestCore : base(lifecycle) { } + public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup) { LogStart($"Starting dashboard server"); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 722bb3f..20023e2 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -67,6 +67,7 @@ namespace Tests.BasicTests .EnableMetrics() .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3)) .WithStorageQuota(3.GB())); var nodes = group.Cast().ToArray(); From 86bae93e980ae9bd471f3642869da795f271078f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 14 Aug 2023 16:37:31 +0200 Subject: [PATCH 20/31] Wires up block-maintenance number of blocks param --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 ++++ DistTestCore/Codex/CodexStartupConfig.cs | 1 + DistTestCore/CodexSetup.cs | 7 +++++++ Tests/BasicTests/ContinuousSubstitute.cs | 1 + 4 files changed, 13 insertions(+) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 0a35afe..99e46f7 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -55,6 +55,10 @@ namespace DistTestCore.Codex { AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString()); } + if (config.BlockMaintenanceNumber != null) + { + AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!); + } if (config.MetricsMode != Metrics.MetricsMode.None) { AddEnvVar("CODEX_METRICS", "true"); diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 3dc9218..c09f066 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -20,5 +20,6 @@ namespace DistTestCore.Codex public string? BootstrapSpr { get; set; } public int? BlockTTL { get; set; } public TimeSpan? BlockMaintenanceInterval { get; set; } + public int? BlockMaintenanceNumber { get; set; } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 1eb99d7..da5fa64 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -12,6 +12,7 @@ namespace DistTestCore ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup WithBlockTTL(TimeSpan duration); ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); + ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks); ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); @@ -64,6 +65,12 @@ namespace DistTestCore return this; } + public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks) + { + BlockMaintenanceNumber = numberOfBlocks; + return this; + } + public ICodexSetup EnableMetrics() { MetricsMode = Metrics.MetricsMode.Record; diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 20023e2..0f83651 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -68,6 +68,7 @@ namespace Tests.BasicTests .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3)) + .WithBlockMaintenanceNumber(10000) .WithStorageQuota(3.GB())); var nodes = group.Cast().ToArray(); From 72fa368357ed193a01f18a605821db20cefc9678 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 15 Aug 2023 10:03:01 +0200 Subject: [PATCH 21/31] working example of crash log recovery --- DistTestCore/Metrics/dashboard.json | 38 +++++++-------- KubernetesWorkflow/K8sController.cs | 3 +- Tests/BasicTests/ContinuousSubstitute.cs | 62 ++++++++++++++---------- 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index fdc5a2a..3af1c52 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -104,7 +104,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -196,7 +196,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -301,7 +301,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -394,7 +394,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -489,7 +489,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -594,7 +594,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -686,7 +686,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -778,7 +778,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -870,7 +870,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -962,7 +962,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1054,7 +1054,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1159,7 +1159,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1251,7 +1251,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1343,7 +1343,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1435,7 +1435,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1527,7 +1527,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1619,7 +1619,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1711,7 +1711,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1803,7 +1803,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 29147ea..9cc8ae7 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -621,8 +621,9 @@ namespace KubernetesWorkflow if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) { log.Log("Pod crash detected for " + container.Name); - using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true)); + using var stream = myOwnClient.ReadNamespacedPodLog(podName, K8sTestNamespace, recipe.Name, previous: true); logHandler.Log(stream); + return; } } }); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0f83651..c522d4a 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,6 +1,7 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; +using System.ComponentModel; using Utils; namespace Tests.BasicTests @@ -60,16 +61,15 @@ namespace Tests.BasicTests } [Test] - [UseLongTimeouts] public void HoldMyBeerTest() { - var group = SetupCodexNodes(5, o => o + var group = SetupCodexNodes(2, o => o .EnableMetrics() - .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(10)) .WithBlockMaintenanceNumber(10000) - .WithStorageQuota(3.GB())); + .WithStorageQuota(500.MB())); var nodes = group.Cast().ToArray(); @@ -79,29 +79,41 @@ namespace Tests.BasicTests try { - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } + //foreach (var node in nodes) + //{ + // node.Marketplace.MakeStorageAvailable( + // size: 1.GB(), + // minPricePerBytePerSecond: 1.TestTokens(), + // maxCollateral: 1024.TestTokens(), + // maxDuration: TimeSpan.FromMinutes(5)); + //} - var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); - while (DateTime.UtcNow < endTime) - { - foreach (var node in nodes) - { - var file = GenerateTestFile(80.MB()); - var cid = node.UploadFile(file); + Thread.Sleep(2000); - var dl = node.DownloadContent(cid); - file.AssertIsEqual(dl); - } + Log("calling crash..."); + var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); + var str = http.HttpGetString("debug/crash"); - Thread.Sleep(TimeSpan.FromMinutes(2)); - } + Log("crash called."); + + Thread.Sleep(TimeSpan.FromSeconds(60)); + + Log("test done."); + + //var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + //while (DateTime.UtcNow < endTime) + //{ + // foreach (var node in nodes) + // { + // var file = GenerateTestFile(80.MB()); + // var cid = node.UploadFile(file); + + // var dl = node.DownloadContent(cid); + // file.AssertIsEqual(dl); + // } + + // Thread.Sleep(TimeSpan.FromSeconds(30)); + //} } finally { From 08ee09af1ef258dcae122bf5bcca9452ab0c7c70 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 15 Aug 2023 11:01:18 +0200 Subject: [PATCH 22/31] wires crashWatcher into CodexStarter --- DistTestCore/Codex/CodexAccess.cs | 29 +++++++- DistTestCore/CodexStarter.cs | 24 ++++++- KubernetesWorkflow/CrashWatcher.cs | 85 ++++++++++++++++++++++++ KubernetesWorkflow/K8sController.cs | 24 +------ KubernetesWorkflow/RunningContainers.cs | 6 +- KubernetesWorkflow/StartupWorkflow.cs | 4 +- Tests/BasicTests/ContinuousSubstitute.cs | 67 ++++++++----------- 7 files changed, 171 insertions(+), 68 deletions(-) create mode 100644 KubernetesWorkflow/CrashWatcher.cs diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index e2158ee..076a335 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -4,10 +4,11 @@ using Utils; namespace DistTestCore.Codex { - public class CodexAccess + public class CodexAccess : ILogHandler { private readonly BaseLog log; private readonly ITimeSet timeSet; + private bool hasContainerCrashed; public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address) { @@ -15,6 +16,9 @@ namespace DistTestCore.Codex Container = container; this.timeSet = timeSet; Address = address; + hasContainerCrashed = false; + + if (container.CrashWatcher != null) container.CrashWatcher.Start(this); } public RunningContainer Container { get; } @@ -86,7 +90,30 @@ namespace DistTestCore.Codex private Http Http() { + CheckContainerCrashed(); return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name); } + + private void CheckContainerCrashed() + { + if (hasContainerCrashed) throw new Exception("Container has crashed."); + } + + public void Log(Stream crashLog) + { + var file = log.CreateSubfile(); + log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'..."); + + using var reader = new StreamReader(crashLog); + var line = reader.ReadLine(); + while (line != null) + { + file.Write(line); + line = reader.ReadLine(); + } + + log.Log("Crash log successfully downloaded."); + hasContainerCrashed = true; + } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 14278c7..b57f9e6 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -47,7 +47,11 @@ namespace DistTestCore { LogStart($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); - foreach (var c in group.Containers) workflow.Stop(c); + foreach (var c in group.Containers) + { + StopCrashWatcher(c); + workflow.Stop(c); + } RunningGroups.Remove(group); LogEnd("Stopped."); } @@ -96,7 +100,9 @@ namespace DistTestCore for (var i = 0; i < numberOfNodes; i++) { var workflow = CreateWorkflow(); - result.Add(workflow.Start(1, location, recipe, startupConfig)); + var rc = workflow.Start(1, location, recipe, startupConfig); + CreateCrashWatcher(workflow, rc); + result.Add(rc); } return result.ToArray(); } @@ -134,5 +140,19 @@ namespace DistTestCore { Log("----------------------------------------------------------------------------"); } + + private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc) + { + var c = rc.Containers.Single(); + c.CrashWatcher = workflow.CreateCrashWatcher(c); + } + + private void StopCrashWatcher(RunningContainers containers) + { + foreach (var c in containers.Containers) + { + c.CrashWatcher?.Stop(); + } + } } } diff --git a/KubernetesWorkflow/CrashWatcher.cs b/KubernetesWorkflow/CrashWatcher.cs new file mode 100644 index 0000000..2909017 --- /dev/null +++ b/KubernetesWorkflow/CrashWatcher.cs @@ -0,0 +1,85 @@ +using k8s; +using Logging; + +namespace KubernetesWorkflow +{ + public class CrashWatcher + { + private readonly BaseLog log; + private readonly KubernetesClientConfiguration config; + private readonly string k8sNamespace; + private readonly RunningContainer container; + private ILogHandler? logHandler; + private CancellationTokenSource cts; + private Task? worker; + private Exception? workerException; + + public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container) + { + this.log = log; + this.config = config; + this.k8sNamespace = k8sNamespace; + this.container = container; + cts = new CancellationTokenSource(); + } + + public void Start(ILogHandler logHandler) + { + if (worker != null) throw new InvalidOperationException(); + + this.logHandler = logHandler; + cts = new CancellationTokenSource(); + worker = Task.Run(Worker); + } + + public void Stop() + { + if (worker == null) throw new InvalidOperationException(); + + cts.Cancel(); + worker.Wait(); + worker = null; + + if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException); + } + + private void Worker() + { + try + { + MonitorContainer(cts.Token); + } + catch (Exception ex) + { + workerException = ex; + } + } + + private void MonitorContainer(CancellationToken token) + { + var client = new Kubernetes(config); + while (!token.IsCancellationRequested) + { + token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)); + + var pod = container.Pod; + var recipe = container.Recipe; + var podName = pod.PodInfo.Name; + var podInfo = client.ReadNamespacedPod(podName, k8sNamespace); + + if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) + { + DownloadCrashedContainerLogs(client, podName, recipe); + return; + } + } + } + + private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe) + { + log.Log("Pod crash detected for " + container.Name); + using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true); + logHandler!.Log(stream); + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 9cc8ae7..03b3039 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -604,29 +604,9 @@ namespace KubernetesWorkflow #endregion - public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler) + public CrashWatcher CreateCrashWatcher(RunningContainer container) { - return Task.Run(() => - { - var myOwnClient = new Kubernetes(cluster.GetK8sClientConfig()); - while (!token.IsCancellationRequested) - { - token.WaitHandle.WaitOne(TimeSpan.FromSeconds(3)); - - var pod = container.Pod; - var recipe = container.Recipe; - var podName = pod.PodInfo.Name; - var podInfo = myOwnClient.ReadNamespacedPod(podName, K8sTestNamespace); - - if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) - { - log.Log("Pod crash detected for " + container.Name); - using var stream = myOwnClient.ReadNamespacedPodLog(podName, K8sTestNamespace, recipe.Name, previous: true); - logHandler.Log(stream); - return; - } - } - }); + return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container); } private PodInfo FetchNewPod() diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 333e4cd..3f43a37 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -1,4 +1,5 @@ -using Utils; +using Newtonsoft.Json; +using Utils; namespace KubernetesWorkflow { @@ -39,6 +40,9 @@ namespace KubernetesWorkflow public Port[] ServicePorts { get; } public Address ClusterExternalAddress { get; } public Address ClusterInternalAddress { get; } + + [JsonIgnore] + public CrashWatcher? CrashWatcher { get; set; } } public static class RunningContainersExtensions diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 6f390a0..d9d567f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -37,9 +37,9 @@ namespace KubernetesWorkflow }, pl); } - public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler) + public CrashWatcher CreateCrashWatcher(RunningContainer container) { - return K8s(controller => controller.WatchForCrashLogs(container, token, logHandler)); + return K8s(controller => controller.CreateCrashWatcher(container)); } public void Stop(RunningContainers runningContainers) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index c522d4a..78ed1b1 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,7 +1,6 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; -using System.ComponentModel; using Utils; namespace Tests.BasicTests @@ -73,53 +72,41 @@ namespace Tests.BasicTests var nodes = group.Cast().ToArray(); - var flow = Get().WorkflowCreator.CreateWorkflow(); - var cst = new CancellationTokenSource(); - var tasks = nodes.Select(n => flow.WatchForCrashLogs(n.CodexAccess.Container, cst.Token, this)).ToArray(); + //foreach (var node in nodes) + //{ + // node.Marketplace.MakeStorageAvailable( + // size: 1.GB(), + // minPricePerBytePerSecond: 1.TestTokens(), + // maxCollateral: 1024.TestTokens(), + // maxDuration: TimeSpan.FromMinutes(5)); + //} - try - { - //foreach (var node in nodes) - //{ - // node.Marketplace.MakeStorageAvailable( - // size: 1.GB(), - // minPricePerBytePerSecond: 1.TestTokens(), - // maxCollateral: 1024.TestTokens(), - // maxDuration: TimeSpan.FromMinutes(5)); - //} + Thread.Sleep(2000); - Thread.Sleep(2000); + Log("calling crash..."); + var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); + var str = http.HttpGetString("debug/crash"); - Log("calling crash..."); - var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); - var str = http.HttpGetString("debug/crash"); + Log("crash called."); - Log("crash called."); + Thread.Sleep(TimeSpan.FromSeconds(60)); - Thread.Sleep(TimeSpan.FromSeconds(60)); + Log("test done."); - Log("test done."); + //var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + //while (DateTime.UtcNow < endTime) + //{ + // foreach (var node in nodes) + // { + // var file = GenerateTestFile(80.MB()); + // var cid = node.UploadFile(file); - //var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); - //while (DateTime.UtcNow < endTime) - //{ - // foreach (var node in nodes) - // { - // var file = GenerateTestFile(80.MB()); - // var cid = node.UploadFile(file); + // var dl = node.DownloadContent(cid); + // file.AssertIsEqual(dl); + // } - // var dl = node.DownloadContent(cid); - // file.AssertIsEqual(dl); - // } - - // Thread.Sleep(TimeSpan.FromSeconds(30)); - //} - } - finally - { - cst.Cancel(); - foreach (var t in tasks) t.Wait(); - } + // Thread.Sleep(TimeSpan.FromSeconds(30)); + //} } public void Log(Stream log) From 3abb1770c239db11ae17277dc465c78d6ce64cde Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 15 Aug 2023 12:55:30 +0200 Subject: [PATCH 23/31] Fixes crash log downloads --- KubernetesWorkflow/CrashWatcher.cs | 2 +- Tests/BasicTests/ContinuousSubstitute.cs | 59 +++++++++--------------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/KubernetesWorkflow/CrashWatcher.cs b/KubernetesWorkflow/CrashWatcher.cs index 2909017..86f8ae1 100644 --- a/KubernetesWorkflow/CrashWatcher.cs +++ b/KubernetesWorkflow/CrashWatcher.cs @@ -60,7 +60,7 @@ namespace KubernetesWorkflow var client = new Kubernetes(config); while (!token.IsCancellationRequested) { - token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)); + token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1)); var pod = container.Pod; var recipe = container.Recipe; diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 78ed1b1..dd085b2 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,12 +1,11 @@ using DistTestCore; -using KubernetesWorkflow; using NUnit.Framework; using Utils; namespace Tests.BasicTests { [TestFixture] - public class ContinuousSubstitute : AutoBootstrapDistTest, ILogHandler + public class ContinuousSubstitute : AutoBootstrapDistTest { [Test] [UseLongTimeouts] @@ -60,13 +59,14 @@ namespace Tests.BasicTests } [Test] + [UseLongTimeouts] public void HoldMyBeerTest() { - var group = SetupCodexNodes(2, o => o + var group = SetupCodexNodes(5, o => o .EnableMetrics() //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(10)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(5)) .WithBlockMaintenanceNumber(10000) .WithStorageQuota(500.MB())); @@ -81,44 +81,31 @@ namespace Tests.BasicTests // maxDuration: TimeSpan.FromMinutes(5)); //} - Thread.Sleep(2000); + //Thread.Sleep(2000); - Log("calling crash..."); - var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); - var str = http.HttpGetString("debug/crash"); + //Log("calling crash..."); + //var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); + //var str = http.HttpGetString("debug/crash"); - Log("crash called."); + //Log("crash called."); - Thread.Sleep(TimeSpan.FromSeconds(60)); + //Thread.Sleep(TimeSpan.FromSeconds(60)); - Log("test done."); + //Log("test done."); - //var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); - //while (DateTime.UtcNow < endTime) - //{ - // foreach (var node in nodes) - // { - // var file = GenerateTestFile(80.MB()); - // var cid = node.UploadFile(file); - - // var dl = node.DownloadContent(cid); - // file.AssertIsEqual(dl); - // } - - // Thread.Sleep(TimeSpan.FromSeconds(30)); - //} - } - - public void Log(Stream log) - { - Log("Well damn, container crashed. Here's the log:"); - using var reader = new StreamReader(log); - - var line = reader.ReadLine(); - while(line != null) + var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + while (DateTime.UtcNow < endTime) { - Log(line); - line = reader.ReadLine(); + foreach (var node in nodes) + { + var file = GenerateTestFile(80.MB()); + var cid = node.UploadFile(file); + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + } + + Thread.Sleep(TimeSpan.FromSeconds(60)); } } } From 57c46004b1a80eace18331d786f63a40f449c6ad Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 15 Aug 2023 15:00:47 +0200 Subject: [PATCH 24/31] cleanup --- Tests/BasicTests/ContinuousSubstitute.cs | 45 +++++++++--------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index dd085b2..0436bcf 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -64,48 +64,35 @@ namespace Tests.BasicTests { var group = SetupCodexNodes(5, o => o .EnableMetrics() - //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(5)) .WithBlockMaintenanceNumber(10000) - .WithStorageQuota(500.MB())); + .WithStorageQuota(1000.MB())); var nodes = group.Cast().ToArray(); - //foreach (var node in nodes) - //{ - // node.Marketplace.MakeStorageAvailable( - // size: 1.GB(), - // minPricePerBytePerSecond: 1.TestTokens(), - // maxCollateral: 1024.TestTokens(), - // maxDuration: TimeSpan.FromMinutes(5)); - //} - - //Thread.Sleep(2000); - - //Log("calling crash..."); - //var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); - //var str = http.HttpGetString("debug/crash"); - - //Log("crash called."); - - //Thread.Sleep(TimeSpan.FromSeconds(60)); - - //Log("test done."); - - var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); while (DateTime.UtcNow < endTime) { foreach (var node in nodes) { - var file = GenerateTestFile(80.MB()); - var cid = node.UploadFile(file); + try + { + var file = GenerateTestFile(80.MB()); + var cid = node.UploadFile(file); - var dl = node.DownloadContent(cid); - file.AssertIsEqual(dl); + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + } + catch + { + Log("Test failed. Delaying shut-down by 30 seconds to collect metrics."); + Thread.Sleep(TimeSpan.FromSeconds(30)); + throw; + } } - Thread.Sleep(TimeSpan.FromSeconds(60)); + Thread.Sleep(TimeSpan.FromSeconds(3)); } } } From 6486dba2895900d1181e5992deba2fb7bdfe6258 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Aug 2023 10:52:44 +0200 Subject: [PATCH 25/31] Asserting correct filesize and block numbers in the codex logs --- DistTestCore/Logs/DownloadedLog.cs | 27 +++++++++ DistTestCore/Timing.cs | 6 +- Tests/BasicTests/ContinuousSubstitute.cs | 75 +++++++++++++++++++++++- 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Logs/DownloadedLog.cs b/DistTestCore/Logs/DownloadedLog.cs index 9d22c81..606c411 100644 --- a/DistTestCore/Logs/DownloadedLog.cs +++ b/DistTestCore/Logs/DownloadedLog.cs @@ -6,6 +6,8 @@ namespace DistTestCore.Logs public interface IDownloadedLog { void AssertLogContains(string expectedString); + string[] FindLinesThatContain(params string[] tags); + void DeleteFile(); } public class DownloadedLog : IDownloadedLog @@ -33,5 +35,30 @@ namespace DistTestCore.Logs Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } + + public string[] FindLinesThatContain(params string[] tags) + { + var result = new List(); + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + if (tags.All(line.Contains)) + { + result.Add(line); + } + + line = streamReader.ReadLine(); + } + + return result.ToArray(); + } + + public void DeleteFile() + { + File.Delete(logFile.FullFilename); + } } } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index bd385ae..38df6d8 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -21,7 +21,7 @@ namespace DistTestCore { public TimeSpan HttpCallTimeout() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromMinutes(5); } public TimeSpan HttpCallRetryTime() @@ -36,12 +36,12 @@ namespace DistTestCore public TimeSpan WaitForK8sServiceDelay() { - return TimeSpan.FromSeconds(1); + return TimeSpan.FromSeconds(10); } public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(1); + return TimeSpan.FromMinutes(30); } public TimeSpan WaitForMetricTimeout() diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0436bcf..4325d57 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using NUnit.Framework.Constraints; using Utils; namespace Tests.BasicTests @@ -59,7 +60,6 @@ namespace Tests.BasicTests } [Test] - [UseLongTimeouts] public void HoldMyBeerTest() { var group = SetupCodexNodes(5, o => o @@ -72,17 +72,52 @@ namespace Tests.BasicTests var nodes = group.Cast().ToArray(); var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); + + var filesize = 80.MB(); + double codexDefaultBlockSize = 31 * 64 * 33; + var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); + var sizeInBytes = filesize.SizeInBytes; + Assert.That(numberOfBlocks, Is.EqualTo(1282)); + while (DateTime.UtcNow < endTime) { foreach (var node in nodes) { try { - var file = GenerateTestFile(80.MB()); + var file = GenerateTestFile(filesize); var cid = node.UploadFile(file); + var cidTag = cid.Id.Substring(cid.Id.Length - (1 + 6)); + var uploadLog = node.DownloadLog(); + + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + if (storeLine == null) + { + Assert.Fail("Storeline not found for cid" + cidTag); + return; + } + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + + var dl = node.DownloadContent(cid); file.AssertIsEqual(dl); + var downloadLog = node.DownloadLog(); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + if (sentLine == null) + { + Assert.Fail("Sentline not found for cid" + cidTag); + return; + } + AssertSentLineContains(sentLine, sizeInBytes); + } catch { @@ -95,5 +130,41 @@ namespace Tests.BasicTests Thread.Sleep(TimeSpan.FromSeconds(3)); } } + + private void AssertSentLineContains(string sentLine, long sizeInBytes) + { + var tag = "bytes="; + var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); + var bytes = Convert.ToInt64(token); + Assert.AreEqual(sizeInBytes, bytes, "Sent bytes: Number of bytes incorrect"); + } + + private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) + { + var tokens = storeLine.Split(" "); + + var blocksToken = GetToken(tokens, "blocks="); + var sizeToken = GetToken(tokens, "size="); + if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); + if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); + + var blocks = Convert.ToInt64(blocksToken); + var size = Convert.ToInt64(sizeToken); + + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect"); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect"); + } + + private string? GetLineForCidTag(string[] lines, string cidTag) + { + return lines.SingleOrDefault(l => l.Contains(cidTag)); + } + + private string? GetToken(string[] tokens, string tag) + { + var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); + if (token == null) return null; + return token.Substring(tag.Length); + } } } From 30364abf2187e5cf384d0a58092731a6b91ed90e Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Aug 2023 11:39:24 +0200 Subject: [PATCH 26/31] Correct checking of upload and download log statements --- Tests/BasicTests/ContinuousSubstitute.cs | 35 ++++++++++-------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 4325d57..6d6a60e 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,6 +1,5 @@ using DistTestCore; using NUnit.Framework; -using NUnit.Framework.Constraints; using Utils; namespace Tests.BasicTests @@ -88,21 +87,15 @@ namespace Tests.BasicTests var file = GenerateTestFile(filesize); var cid = node.UploadFile(file); - var cidTag = cid.Id.Substring(cid.Id.Length - (1 + 6)); + var cidTag = cid.Id.Substring(cid.Id.Length - 6); var uploadLog = node.DownloadLog(); var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); uploadLog.DeleteFile(); var storeLine = GetLineForCidTag(storeLines, cidTag); - if (storeLine == null) - { - Assert.Fail("Storeline not found for cid" + cidTag); - return; - } AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); - var dl = node.DownloadContent(cid); file.AssertIsEqual(dl); var downloadLog = node.DownloadLog(); @@ -111,13 +104,7 @@ namespace Tests.BasicTests downloadLog.DeleteFile(); var sentLine = GetLineForCidTag(sentLines, cidTag); - if (sentLine == null) - { - Assert.Fail("Sentline not found for cid" + cidTag); - return; - } AssertSentLineContains(sentLine, sizeInBytes); - } catch { @@ -136,7 +123,7 @@ namespace Tests.BasicTests var tag = "bytes="; var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); var bytes = Convert.ToInt64(token); - Assert.AreEqual(sizeInBytes, bytes, "Sent bytes: Number of bytes incorrect"); + Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); } private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) @@ -149,15 +136,23 @@ namespace Tests.BasicTests if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); var blocks = Convert.ToInt64(blocksToken); - var size = Convert.ToInt64(sizeToken); + var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); - Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect"); - Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect"); + var lineLog = $" Line: '{storeLine}'"; + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); } - private string? GetLineForCidTag(string[] lines, string cidTag) + private string GetLineForCidTag(string[] lines, string cidTag) { - return lines.SingleOrDefault(l => l.Contains(cidTag)); + var result = lines.SingleOrDefault(l => l.Contains(cidTag)); + if (result == null) + { + Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); + throw new Exception(); + } + + return result; } private string? GetToken(string[] tokens, string tag) From c8cb04d859c098fd46e7a0a5ab43713416c0ebaa Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Aug 2023 16:13:29 +0200 Subject: [PATCH 27/31] Allows downloading only log-tails. --- ContinuousTests/ContinuousLogDownloader.cs | 2 +- DistTestCore/CodexStarter.cs | 4 +- .../Marketplace/CodexContractsStarter.cs | 2 +- .../Marketplace/ContainerInfoExtractor.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 6 +-- DistTestCore/PrometheusStarter.cs | 2 +- DistTestCore/TestLifecycle.cs | 4 +- KubernetesWorkflow/K8sController.cs | 4 +- KubernetesWorkflow/StartupWorkflow.cs | 4 +- Tests/BasicTests/ContinuousSubstitute.cs | 52 +++++++++++++------ 10 files changed, 52 insertions(+), 30 deletions(-) diff --git a/ContinuousTests/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs index 6e387a1..7c3cb50 100644 --- a/ContinuousTests/ContinuousLogDownloader.cs +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -55,7 +55,7 @@ namespace ContinuousTests var appender = new LogAppender(filepath); - lifecycle.CodexStarter.DownloadLog(container, appender); + lifecycle.CodexStarter.DownloadLog(container, appender, null); } private static string GetLogName(RunningContainer container) diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index b57f9e6..86a839e 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -64,10 +64,10 @@ namespace DistTestCore RunningGroups.Clear(); } - public void DownloadLog(RunningContainer container, ILogHandler logHandler) + public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines) { var workflow = CreateWorkflow(); - workflow.DownloadContainerLog(container, logHandler); + workflow.DownloadContainerLog(container, logHandler, tailLines); } private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index a1db57f..4d628b6 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace WaitUntil(() => { var logHandler = new ContractsReadyLogHandler(Debug); - workflow.DownloadContainerLog(container, logHandler); + workflow.DownloadContainerLog(container, logHandler, null); return logHandler.Found; }); Log("Contracts deployed. Extracting addresses..."); diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 7fd5638..1eac5cb 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace private string FetchPubKey() { var enodeFinder = new PubKeyFinder(s => log.Debug(s)); - workflow.DownloadContainerLog(container, enodeFinder); + workflow.DownloadContainerLog(container, enodeFinder, null); return enodeFinder.GetPubKey(); } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 572cb9f..52fb81f 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -16,7 +16,7 @@ namespace DistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); - IDownloadedLog DownloadLog(); + IDownloadedLog DownloadLog(int? tailLines = null); IMetricsAccess Metrics { get; } IMarketplaceAccess Marketplace { get; } CodexDebugVersionResponse Version { get; } @@ -103,9 +103,9 @@ namespace DistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - public IDownloadedLog DownloadLog() + public IDownloadedLog DownloadLog(int? tailLines = null) { - return lifecycle.DownloadLog(CodexAccess.Container); + return lifecycle.DownloadLog(CodexAccess.Container, tailLines); } public ICodexSetup BringOffline() diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 3b1e49c..f19978c 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -29,7 +29,7 @@ namespace DistTestCore { var config = ""; config += "global:\n"; - config += " scrape_interval: 30s\n"; + config += " scrape_interval: 10s\n"; config += " scrape_timeout: 10s\n"; config += "\n"; config += "scrape_configs:\n"; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 260e438..501c5cb 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -49,14 +49,14 @@ namespace DistTestCore FileManager.DeleteAllTestFiles(); } - public IDownloadedLog DownloadLog(RunningContainer container) + public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) { var subFile = Log.CreateSubfile(); var description = container.Name; var handler = new LogDownloadHandler(container, description, subFile); Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); - CodexStarter.DownloadLog(container, handler); + CodexStarter.DownloadLog(container, handler, tailLines); return new DownloadedLog(subFile, description); } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 03b3039..7ca195b 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -53,10 +53,10 @@ namespace KubernetesWorkflow WaitUntilPodOffline(pod.PodInfo.Name); } - public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) + public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines) { log.Debug(); - using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name)); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines)); logHandler.Log(stream); } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index d9d567f..b2ae792 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -50,11 +50,11 @@ namespace KubernetesWorkflow }); } - public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler) + public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines) { K8s(controller => { - controller.DownloadPodLog(container.Pod, container.Recipe, logHandler); + controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines); }); } diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 6d6a60e..36f071e 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,4 +1,5 @@ using DistTestCore; +using Logging; using NUnit.Framework; using Utils; @@ -61,12 +62,13 @@ namespace Tests.BasicTests [Test] public void HoldMyBeerTest() { - var group = SetupCodexNodes(5, o => o + var blockExpirationTime = TimeSpan.FromMinutes(3); + var group = SetupCodexNodes(1, o => o .EnableMetrics() - .WithBlockTTL(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(5)) + .WithBlockTTL(blockExpirationTime) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(1)) .WithBlockMaintenanceNumber(10000) - .WithStorageQuota(1000.MB())); + .WithStorageQuota(2000.MB())); var nodes = group.Cast().ToArray(); @@ -78,43 +80,63 @@ namespace Tests.BasicTests var sizeInBytes = filesize.SizeInBytes; Assert.That(numberOfBlocks, Is.EqualTo(1282)); + var successfulUploads = 0; + var successfulDownloads = 0; + while (DateTime.UtcNow < endTime) { foreach (var node in nodes) { try { + var uploadStartTime = DateTime.UtcNow; var file = GenerateTestFile(filesize); var cid = node.UploadFile(file); var cidTag = cid.Id.Substring(cid.Id.Length - 6); - var uploadLog = node.DownloadLog(); + Stopwatch.Measure(Get().Log, "upload-log-asserts", () => + { + var uploadLog = node.DownloadLog(tailLines: 50000); - var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); - uploadLog.DeleteFile(); + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); - var storeLine = GetLineForCidTag(storeLines, cidTag); - AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + var storeLine = GetLineForCidTag(storeLines, cidTag); + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + }); + successfulUploads++; + + var uploadTimeTaken = DateTime.UtcNow - uploadStartTime; + if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10))) + { + Assert.Fail("Upload took too long. Blocks already expired."); + } var dl = node.DownloadContent(cid); file.AssertIsEqual(dl); - var downloadLog = node.DownloadLog(); - var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); - downloadLog.DeleteFile(); + Stopwatch.Measure(Get().Log, "download-log-asserts", () => + { + var downloadLog = node.DownloadLog(tailLines: 50000); - var sentLine = GetLineForCidTag(sentLines, cidTag); - AssertSentLineContains(sentLine, sizeInBytes); + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + AssertSentLineContains(sentLine, sizeInBytes); + }); + successfulDownloads++; } catch { Log("Test failed. Delaying shut-down by 30 seconds to collect metrics."); + Log($"Test failed after {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); Thread.Sleep(TimeSpan.FromSeconds(30)); throw; } } - Thread.Sleep(TimeSpan.FromSeconds(3)); + Thread.Sleep(TimeSpan.FromSeconds(5)); } } From 1682dfc08fadd000b9af2904d9ef76a447a6ea22 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 18 Aug 2023 10:31:20 +0200 Subject: [PATCH 28/31] Adds holdmybeer test to continuous test runner --- ContinuousTests/NodeRunner.cs | 16 ++++ ContinuousTests/Tests/HoldMyBeerTest.cs | 103 +++++++++++++++++++++++ Tests/BasicTests/ContinuousSubstitute.cs | 4 +- 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 ContinuousTests/Tests/HoldMyBeerTest.cs diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 5b79bf5..d599c4d 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -5,6 +5,7 @@ using KubernetesWorkflow; using NUnit.Framework; using Logging; using Utils; +using DistTestCore.Logs; namespace ContinuousTests { @@ -38,6 +39,21 @@ namespace ContinuousTests RunNode(bootstrapNode, operation, 0.TestTokens()); } + public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null) + { + var subFile = log.CreateSubfile(); + var description = container.Name; + var handler = new LogDownloadHandler(container, description, subFile); + + log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); + + var lifecycle = CreateTestLifecycle(); + var flow = lifecycle.WorkflowCreator.CreateWorkflow(); + flow.DownloadContainerLog(container, handler, tailLines); + + return new DownloadedLog(subFile, description); + } + public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { var lifecycle = CreateTestLifecycle(); diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs new file mode 100644 index 0000000..0f5e422 --- /dev/null +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -0,0 +1,103 @@ +using DistTestCore; +using Logging; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class HoldMyBeerTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 1; + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + + private ContentId? cid; + private TestFile file = null!; + + [TestMoment(t: Zero)] + public void UploadTestFile() + { + var filesize = 80.MB(); + double codexDefaultBlockSize = 31 * 64 * 33; + var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); + var sizeInBytes = filesize.SizeInBytes; + Assert.That(numberOfBlocks, Is.EqualTo(1282)); + + + file = FileManager.GenerateTestFile(filesize); + + cid = UploadFile(Nodes[0], file); + Assert.That(cid, Is.Not.Null); + + var cidTag = cid!.Id.Substring(cid.Id.Length - 6); + Stopwatch.Measure(Log, "upload-log-asserts", () => + { + var uploadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000); + + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + }); + + + var dl = DownloadFile(Nodes[1], cid!); + file.AssertIsEqual(dl); + + Stopwatch.Measure(Log, "download-log-asserts", () => + { + var downloadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + AssertSentLineContains(sentLine, sizeInBytes); + }); + } + + private void AssertSentLineContains(string sentLine, long sizeInBytes) + { + var tag = "bytes="; + var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); + var bytes = Convert.ToInt64(token); + Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); + } + + private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) + { + var tokens = storeLine.Split(" "); + + var blocksToken = GetToken(tokens, "blocks="); + var sizeToken = GetToken(tokens, "size="); + if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); + if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); + + var blocks = Convert.ToInt64(blocksToken); + var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); + + var lineLog = $" Line: '{storeLine}'"; + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); + } + + private string GetLineForCidTag(string[] lines, string cidTag) + { + var result = lines.SingleOrDefault(l => l.Contains(cidTag)); + if (result == null) + { + Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); + throw new Exception(); + } + + return result; + } + + private string? GetToken(string[] tokens, string tag) + { + var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); + if (token == null) return null; + return token.Substring(tag.Length); + } + } +} diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 36f071e..0dc0113 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -80,6 +80,7 @@ namespace Tests.BasicTests var sizeInBytes = filesize.SizeInBytes; Assert.That(numberOfBlocks, Is.EqualTo(1282)); + var startTime = DateTime.UtcNow; var successfulUploads = 0; var successfulDownloads = 0; @@ -129,8 +130,9 @@ namespace Tests.BasicTests } catch { + var testDuration = DateTime.UtcNow - startTime; Log("Test failed. Delaying shut-down by 30 seconds to collect metrics."); - Log($"Test failed after {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); + Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); Thread.Sleep(TimeSpan.FromSeconds(30)); throw; } From ce3850b0b0a23b0f88f01a6ee00b832ca53a0121 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 21 Aug 2023 07:58:58 +0200 Subject: [PATCH 29/31] I accidentally filled up my harddrive with random numbers last night. --- Tests/BasicTests/ContinuousSubstitute.cs | 75 +++++++++++++----------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0dc0113..1723b70 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -63,8 +63,8 @@ namespace Tests.BasicTests public void HoldMyBeerTest() { var blockExpirationTime = TimeSpan.FromMinutes(3); - var group = SetupCodexNodes(1, o => o - .EnableMetrics() + var group = SetupCodexNodes(3, o => o + //.EnableMetrics() .WithBlockTTL(blockExpirationTime) .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(1)) .WithBlockMaintenanceNumber(10000) @@ -72,7 +72,7 @@ namespace Tests.BasicTests var nodes = group.Cast().ToArray(); - var endTime = DateTime.UtcNow + TimeSpan.FromHours(1); + var endTime = DateTime.UtcNow + TimeSpan.FromHours(24); var filesize = 80.MB(); double codexDefaultBlockSize = 31 * 64 * 33; @@ -90,43 +90,48 @@ namespace Tests.BasicTests { try { - var uploadStartTime = DateTime.UtcNow; - var file = GenerateTestFile(filesize); - var cid = node.UploadFile(file); + Thread.Sleep(TimeSpan.FromSeconds(5)); - var cidTag = cid.Id.Substring(cid.Id.Length - 6); - Stopwatch.Measure(Get().Log, "upload-log-asserts", () => + ScopedTestFiles(() => { - var uploadLog = node.DownloadLog(tailLines: 50000); + var uploadStartTime = DateTime.UtcNow; + var file = GenerateTestFile(filesize); + var cid = node.UploadFile(file); - var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); - uploadLog.DeleteFile(); + var cidTag = cid.Id.Substring(cid.Id.Length - 6); + Stopwatch.Measure(Get().Log, "upload-log-asserts", () => + { + var uploadLog = node.DownloadLog(tailLines: 50000); - var storeLine = GetLineForCidTag(storeLines, cidTag); - AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + }); + successfulUploads++; + + var uploadTimeTaken = DateTime.UtcNow - uploadStartTime; + if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10))) + { + Assert.Fail("Upload took too long. Blocks already expired."); + } + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + + Stopwatch.Measure(Get().Log, "download-log-asserts", () => + { + var downloadLog = node.DownloadLog(tailLines: 50000); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + AssertSentLineContains(sentLine, sizeInBytes); + }); + successfulDownloads++; }); - successfulUploads++; - - var uploadTimeTaken = DateTime.UtcNow - uploadStartTime; - if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10))) - { - Assert.Fail("Upload took too long. Blocks already expired."); - } - - var dl = node.DownloadContent(cid); - file.AssertIsEqual(dl); - - Stopwatch.Measure(Get().Log, "download-log-asserts", () => - { - var downloadLog = node.DownloadLog(tailLines: 50000); - - var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); - downloadLog.DeleteFile(); - - var sentLine = GetLineForCidTag(sentLines, cidTag); - AssertSentLineContains(sentLine, sizeInBytes); - }); - successfulDownloads++; } catch { From 7e5121a642ca2ec2c057d46ab5857d03929be76d Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 22 Aug 2023 09:06:44 +0200 Subject: [PATCH 30/31] Fixes test for cloud environment --- ContinuousTests/Tests/HoldMyBeerTest.cs | 2 +- Tests/BasicTests/ContinuousSubstitute.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index 0f5e422..a755e9d 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -41,7 +41,7 @@ namespace ContinuousTests.Tests }); - var dl = DownloadFile(Nodes[1], cid!); + var dl = DownloadFile(Nodes[0], cid!); file.AssertIsEqual(dl); Stopwatch.Measure(Log, "download-log-asserts", () => diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 1723b70..5b18c34 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -64,9 +64,9 @@ namespace Tests.BasicTests { var blockExpirationTime = TimeSpan.FromMinutes(3); var group = SetupCodexNodes(3, o => o - //.EnableMetrics() + .EnableMetrics() .WithBlockTTL(blockExpirationTime) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(1)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) .WithBlockMaintenanceNumber(10000) .WithStorageQuota(2000.MB())); From 47aedc2ac864f756e9eef10c63c8f0d0aa777c9a Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 22 Aug 2023 15:51:39 +0200 Subject: [PATCH 31/31] cleanup --- ContinuousTests/ContinuousLogDownloader.cs | 2 -- ContinuousTests/ContinuousTestRunner.cs | 3 ++- DistTestCore/DistTest.cs | 7 ++++++- Tests/BasicTests/ContinuousSubstitute.cs | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ContinuousTests/ContinuousLogDownloader.cs b/ContinuousTests/ContinuousLogDownloader.cs index 7c3cb50..60a2882 100644 --- a/ContinuousTests/ContinuousLogDownloader.cs +++ b/ContinuousTests/ContinuousLogDownloader.cs @@ -8,7 +8,6 @@ namespace ContinuousTests { private readonly TestLifecycle lifecycle; private readonly RunningContainer[] containers; - //private readonly CodexDeployment deployment; private readonly string outputPath; private readonly CancellationToken cancelToken; @@ -16,7 +15,6 @@ namespace ContinuousTests { this.lifecycle = lifecycle; this.containers = containers; - //this.deployment = deployment; this.outputPath = outputPath; this.cancelToken = cancelToken; } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 61ecbbf..2f82cbb 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -30,7 +30,8 @@ namespace ContinuousTests ClearAllCustomNamespaces(allTests, overviewLog); - StartLogDownloader(taskFactory); + // Disabled for now. Still looking for a better way. + // StartLogDownloader(taskFactory); var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray(); diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index f02e620..943b098 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -175,12 +175,17 @@ namespace DistTestCore GetTestLog().Debug(msg); } + public void Measure(string name, Action action) + { + Stopwatch.Measure(Get().Log, name, action); + } + protected CodexSetup CreateCodexSetup(int numberOfNodes) { return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel()); } - public TestLifecycle Get() + private TestLifecycle Get() { lock (lifecycleLock) { diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 5b18c34..2ce65c8 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -99,7 +99,7 @@ namespace Tests.BasicTests var cid = node.UploadFile(file); var cidTag = cid.Id.Substring(cid.Id.Length - 6); - Stopwatch.Measure(Get().Log, "upload-log-asserts", () => + Measure("upload-log-asserts", () => { var uploadLog = node.DownloadLog(tailLines: 50000); @@ -120,7 +120,7 @@ namespace Tests.BasicTests var dl = node.DownloadContent(cid); file.AssertIsEqual(dl); - Stopwatch.Measure(Get().Log, "download-log-asserts", () => + Measure("download-log-asserts", () => { var downloadLog = node.DownloadLog(tailLines: 50000);