status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automatic quota line. Holding test.

This commit is contained in:
benbierens 2023-08-14 10:26:04 +02:00
parent f2346d5d09
commit 9f0600d6c1
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
6 changed files with 106 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CodexNetDeployer/Deployer.cs
View File

@ -85,7 +85,7 @@ namespace CodexNetDeployer
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
Log("Starting dashboard service...");
var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer);
var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
return (prometheusContainer, grafanaStartInfo);
}

2
DistTestCore/CodexStarter.cs
View File

@ -74,7 +74,7 @@ namespace DistTestCore
if (codexSetup.MetricsMode == MetricsMode.Dashboard)
{
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First());
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
}
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);

45
DistTestCore/GrafanaStarter.cs
View File

@ -8,11 +8,14 @@ namespace DistTestCore
{
public class GrafanaStarter : BaseStarter
{
private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
public GrafanaStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer)
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
{
LogStart($"Starting dashboard server");
@ -25,7 +28,7 @@ namespace DistTestCore
AddDataSource(http, prometheusContainer);
Log("Uploading dashboard configurations...");
var jsons = ReadEachDashboardJsonFile();
var jsons = ReadEachDashboardJsonFile(codexSetup);
var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
LogEnd("Dashboard server started.");
@ -86,7 +89,7 @@ namespace DistTestCore
return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
}
private static string[] ReadEachDashboardJsonFile()
private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceNames = new[]
@ -94,15 +97,45 @@ namespace DistTestCore
"DistTestCore.Metrics.dashboard.json"
};
return resourceNames.Select(r => GetManifestResource(assembly, r)).ToArray();
return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
}
private static string GetManifestResource(Assembly assembly, string resourceName)
private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
{
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream == null) throw new Exception("Unable to find resource " + resourceName);
using var reader = new StreamReader(stream);
return reader.ReadToEnd();
return ApplyReplacements(reader.ReadToEnd(), codexSetup);
}
private static string ApplyReplacements(string input, CodexSetup codexSetup)
{
var quotaString = GetQuotaString(codexSetup);
var softMaxString = GetSoftMaxString(codexSetup);
return input
.Replace(StorageQuotaThresholdReplaceToken, quotaString)
.Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
}
private static string GetQuotaString(CodexSetup codexSetup)
{
return GetCodexStorageQuotaInBytes(codexSetup).ToString();
}
private static string GetSoftMaxString(CodexSetup codexSetup)
{
var quota = GetCodexStorageQuotaInBytes(codexSetup);
var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
return softMax.ToString();
}
private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
{
if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
// Codex default: 8GB
return 8.GB().SizeInBytes;
}
private static string GetDashboardCreateRequest(string dashboardJson)

9
DistTestCore/Http.cs
View File

@ -148,14 +148,7 @@ namespace DistTestCore
private T Retry<T>(Func<T> operation, string description)
{
try
{
return operation();
}
catch (Exception ex)
{
throw new Exception(description, ex);
}
return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
}
private HttpClient GetClient()

6
DistTestCore/Metrics/dashboard.json
View File

@ -431,6 +431,8 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": "<CODEX_BYTESUSED_SOFTMAX>",
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
@ -453,7 +455,7 @@
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
"mode": "line"
}
},
"mappings": [],
@ -466,7 +468,7 @@
},
{
"color": "red",
"value": 80
"value": "<CODEX_STORAGEQUOTA>"
}
]
}

103
Tests/BasicTests/ContinuousSubstitute.cs
View File

@ -1,5 +1,4 @@
using ContinuousTests;
using DistTestCore;
using DistTestCore;
using NUnit.Framework;
using Utils;
@ -12,53 +11,34 @@ namespace Tests.BasicTests
[UseLongTimeouts]
public void ContinuousTestSubstitute()
{
var nodes = new List<OnlineCodexNode>();
for (var i = 0; i < 5; i++)
var group = SetupCodexNodes(5, o => o
.EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithStorageQuota(3.GB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray();
foreach (var node in nodes)
{
nodes.Add((OnlineCodexNode)SetupCodexNode(o => o
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: i < 2)
.WithStorageQuota(3.GB())
));
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
var cts = new CancellationTokenSource();
var ct = cts.Token;
var dlPath = Path.Combine(new FileInfo(Get().Log.LogFile.FullFilename)!.Directory!.FullName, "continuouslogs");
Directory.CreateDirectory(dlPath);
var containers = nodes.Select(n => n.CodexAccess.Container).ToArray();
var cd = new ContinuousLogDownloader(Get(), containers, dlPath, ct);
var logTask = Task.Run(cd.Run);
try
var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
var allNodes = nodes.ToList();
var primary = allNodes.PickOneRandom();
var secondary = allNodes.PickOneRandom();
var endTime = DateTime.UtcNow + TimeSpan.FromHours(1);
while (DateTime.UtcNow < endTime)
{
var allNodes = nodes.ToList();
var primary = allNodes.PickOneRandom();
var secondary = allNodes.PickOneRandom();
Log("Run Test");
PerformTest(primary, secondary);
Log("Run Test");
PerformTest(primary, secondary);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
finally
{
cts.Cancel();
logTask.Wait();
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
@ -77,5 +57,42 @@ namespace Tests.BasicTests
testFile.AssertIsEqual(downloadedFile);
});
}
[Test]
[UseLongTimeouts]
public void HoldMyBeerTest()
{
var group = SetupCodexNodes(5, o => o
.EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithStorageQuota(3.GB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray();
foreach (var node in nodes)
{
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
var endTime = DateTime.UtcNow + TimeSpan.FromHours(2);
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
var file = GenerateTestFile(80.MB());
var cid = node.UploadFile(file);
var dl = node.DownloadContent(cid);
file.AssertIsEqual(dl);
}
Thread.Sleep(TimeSpan.FromMinutes(2));
}
}
}
}