status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

working example of crash log recovery

This commit is contained in:
benbierens 2023-08-15 10:03:01 +02:00
parent 86bae93e98
commit 72fa368357
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
3 changed files with 58 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
DistTestCore/Metrics/dashboard.json
View File

@ -104,7 +104,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -196,7 +196,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -301,7 +301,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -394,7 +394,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -489,7 +489,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -594,7 +594,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -686,7 +686,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -778,7 +778,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -870,7 +870,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -962,7 +962,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1054,7 +1054,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1159,7 +1159,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1251,7 +1251,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1343,7 +1343,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1435,7 +1435,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1527,7 +1527,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1619,7 +1619,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1711,7 +1711,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",
@ -1803,7 +1803,7 @@
"calcs": [], "calcs": [],
"displayMode": "list", "displayMode": "list",
"placement": "bottom", "placement": "bottom",
"showLegend": true "showLegend": false
}, },
"tooltip": { "tooltip": {
"mode": "single", "mode": "single",

3
KubernetesWorkflow/K8sController.cs
View File

@ -621,8 +621,9 @@ namespace KubernetesWorkflow
if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
{ {
log.Log("Pod crash detected for " + container.Name); log.Log("Pod crash detected for " + container.Name);
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true)); using var stream = myOwnClient.ReadNamespacedPodLog(podName, K8sTestNamespace, recipe.Name, previous: true);
logHandler.Log(stream); logHandler.Log(stream);
return;
} }
} }
}); });

62
Tests/BasicTests/ContinuousSubstitute.cs
View File

@ -1,6 +1,7 @@
using DistTestCore; using DistTestCore;
using KubernetesWorkflow; using KubernetesWorkflow;
using NUnit.Framework; using NUnit.Framework;
using System.ComponentModel;
using Utils; using Utils;
namespace Tests.BasicTests namespace Tests.BasicTests
@ -60,16 +61,15 @@ namespace Tests.BasicTests
} }
[Test] [Test]
[UseLongTimeouts]
public void HoldMyBeerTest() public void HoldMyBeerTest()
{ {
var group = SetupCodexNodes(5, o => o var group = SetupCodexNodes(2, o => o
.EnableMetrics() .EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2)) .WithBlockTTL(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3)) .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(10))
.WithBlockMaintenanceNumber(10000) .WithBlockMaintenanceNumber(10000)
.WithStorageQuota(3.GB())); .WithStorageQuota(500.MB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray(); var nodes = group.Cast<OnlineCodexNode>().ToArray();
@ -79,29 +79,41 @@ namespace Tests.BasicTests
try try
{ {
foreach (var node in nodes) //foreach (var node in nodes)
{ //{
node.Marketplace.MakeStorageAvailable( // node.Marketplace.MakeStorageAvailable(
size: 1.GB(), // size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(), // minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(), // maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5)); // maxDuration: TimeSpan.FromMinutes(5));
} //}
var endTime = DateTime.UtcNow + TimeSpan.FromHours(2); Thread.Sleep(2000);
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
var file = GenerateTestFile(80.MB());
var cid = node.UploadFile(file);
var dl = node.DownloadContent(cid); Log("calling crash...");
file.AssertIsEqual(dl); var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name);
} var str = http.HttpGetString("debug/crash");
Thread.Sleep(TimeSpan.FromMinutes(2)); Log("crash called.");
}
Thread.Sleep(TimeSpan.FromSeconds(60));
Log("test done.");
//var endTime = DateTime.UtcNow + TimeSpan.FromHours(2);
//while (DateTime.UtcNow < endTime)
//{
// foreach (var node in nodes)
// {
// var file = GenerateTestFile(80.MB());
// var cid = node.UploadFile(file);
// var dl = node.DownloadContent(cid);
// file.AssertIsEqual(dl);
// }
// Thread.Sleep(TimeSpan.FromSeconds(30));
//}
} }
finally finally
{ {