status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

working example of crash log recovery

This commit is contained in:
benbierens 2023-08-15 10:03:01 +02:00
parent 86bae93e98
commit 72fa368357
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
3 changed files with 58 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
DistTestCore/Metrics/dashboard.json
View File

@ -104,7 +104,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -196,7 +196,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -301,7 +301,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -394,7 +394,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -489,7 +489,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -594,7 +594,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -686,7 +686,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -778,7 +778,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -870,7 +870,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -962,7 +962,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1054,7 +1054,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1159,7 +1159,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1251,7 +1251,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1343,7 +1343,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1435,7 +1435,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1527,7 +1527,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1619,7 +1619,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1711,7 +1711,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",
@ -1803,7 +1803,7 @@
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": false
},
"tooltip": {
"mode": "single",

3
KubernetesWorkflow/K8sController.cs
View File

@ -621,8 +621,9 @@ namespace KubernetesWorkflow
if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
{
log.Log("Pod crash detected for " + container.Name);
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true));
using var stream = myOwnClient.ReadNamespacedPodLog(podName, K8sTestNamespace, recipe.Name, previous: true);
logHandler.Log(stream);
return;
}
}
});

62
Tests/BasicTests/ContinuousSubstitute.cs
View File

@ -1,6 +1,7 @@
using DistTestCore;
using KubernetesWorkflow;
using NUnit.Framework;
using System.ComponentModel;
using Utils;
namespace Tests.BasicTests
@ -60,16 +61,15 @@ namespace Tests.BasicTests
}
[Test]
[UseLongTimeouts]
public void HoldMyBeerTest()
{
var group = SetupCodexNodes(5, o => o
var group = SetupCodexNodes(2, o => o
.EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
//.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(3))
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(10))
.WithBlockMaintenanceNumber(10000)
.WithStorageQuota(3.GB()));
.WithStorageQuota(500.MB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray();
@ -79,29 +79,41 @@ namespace Tests.BasicTests
try
{
foreach (var node in nodes)
{
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
//foreach (var node in nodes)
//{
// node.Marketplace.MakeStorageAvailable(
// size: 1.GB(),
// minPricePerBytePerSecond: 1.TestTokens(),
// maxCollateral: 1024.TestTokens(),
// maxDuration: TimeSpan.FromMinutes(5));
//}
var endTime = DateTime.UtcNow + TimeSpan.FromHours(2);
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
var file = GenerateTestFile(80.MB());
var cid = node.UploadFile(file);
Thread.Sleep(2000);
var dl = node.DownloadContent(cid);
file.AssertIsEqual(dl);
}
Log("calling crash...");
var http = new Http(Get().Log, Get().TimeSet, nodes.First().CodexAccess.Address, baseUrl: "/api/codex/v1", nodes.First().CodexAccess.Container.Name);
var str = http.HttpGetString("debug/crash");
Thread.Sleep(TimeSpan.FromMinutes(2));
}
Log("crash called.");
Thread.Sleep(TimeSpan.FromSeconds(60));
Log("test done.");
//var endTime = DateTime.UtcNow + TimeSpan.FromHours(2);
//while (DateTime.UtcNow < endTime)
//{
// foreach (var node in nodes)
// {
// var file = GenerateTestFile(80.MB());
// var cid = node.UploadFile(file);
// var dl = node.DownloadContent(cid);
// file.AssertIsEqual(dl);
// }
// Thread.Sleep(TimeSpan.FromSeconds(30));
//}
}
finally
{