From 72fa368357ed193a01f18a605821db20cefc9678 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 15 Aug 2023 10:03:01 +0200 Subject: [PATCH] working example of crash log recovery --- DistTestCore/Metrics/dashboard.json | 38 +++++++-------- KubernetesWorkflow/K8sController.cs | 3 +- Tests/BasicTests/ContinuousSubstitute.cs | 62 ++++++++++++++---------- 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index fdc5a2a..3af1c52 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -104,7 +104,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -196,7 +196,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -301,7 +301,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -394,7 +394,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -489,7 +489,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -594,7 +594,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -686,7 +686,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -778,7 +778,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -870,7 +870,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -962,7 +962,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1054,7 +1054,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1159,7 +1159,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1251,7 +1251,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1343,7 +1343,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1435,7 +1435,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1527,7 +1527,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1619,7 +1619,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1711,7 +1711,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -1803,7 +1803,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 29147ea..9cc8ae7 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -621,8 +621,9 @@ namespace KubernetesWorkflow if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) { log.Log("Pod crash detected for " + container.Name); - using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true)); + using var stream = myOwnClient.ReadNamespacedPodLog(podName, K8sTestNamespace, recipe.Name, previous: true); logHandler.Log(stream); + return; } } }); diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs index 0f83651..c522d4a 100644 --- a/Tests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -1,6 +1,7 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; +using System.ComponentModel; using Utils; namespace Tests.BasicTests @@ -60,16 +61,15 @@ namespace Tests.BasicTests } [Test] - [UseLongTimeouts] public void HoldMyBeerTest() { - var group = SetupCodexNodes(5, o => o + var group = SetupCodexNodes(2, o => o .EnableMetrics() - .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) .WithBlockTTL(TimeSpan.FromMinutes(2)) - .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(10)) .WithBlockMaintenanceNumber(10000) - .WithStorageQuota(3.GB())); + .WithStorageQuota(500.MB())); var nodes = group.Cast().ToArray(); @@ -79,29 +79,41 @@ namespace Tests.BasicTests try { - foreach (var node in nodes) - { - node.Marketplace.MakeStorageAvailable( - size: 1.GB(), - minPricePerBytePerSecond: 1.TestTokens(), - maxCollateral: 1024.TestTokens(), - maxDuration: TimeSpan.FromMinutes(5)); - } + //foreach (var node in nodes) + //{ + // node.Marketplace.MakeStorageAvailable( + // size: 1.GB(), + // minPricePerBytePerSecond: 1.TestTokens(), + // maxCollateral: 1024.TestTokens(), + // maxDuration: TimeSpan.FromMinutes(5)); + //} - var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); - while (DateTime.UtcNow < endTime) - { - foreach (var node in nodes) - { - var file = GenerateTestFile(80.MB()); - var cid = node.UploadFile(file); + Thread.Sleep(2000); - var dl = node.DownloadContent(cid); - file.AssertIsEqual(dl); - } + Log("calling crash..."); + var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name); + var str = http.HttpGetString("debug/crash"); - Thread.Sleep(TimeSpan.FromMinutes(2)); - } + Log("crash called."); + + Thread.Sleep(TimeSpan.FromSeconds(60)); + + Log("test done."); + + //var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); + //while (DateTime.UtcNow < endTime) + //{ + // foreach (var node in nodes) + // { + // var file = GenerateTestFile(80.MB()); + // var cid = node.UploadFile(file); + + // var dl = node.DownloadContent(cid); + // file.AssertIsEqual(dl); + // } + + // Thread.Sleep(TimeSpan.FromSeconds(30)); + //} } finally {