status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-01-11 09:06:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'spike/local-continuous-debug'

Browse Source
This commit is contained in:
benbierens 2023-08-22 15:56:05 +02:00
commit e340b3cd41
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
41 changed files with 2724 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -84,7 +84,7 @@ namespace CodexNetDeployer
var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
marketplaceConfig.AccountIndexOverride = i;
codexStart.MarketplaceConfig = marketplaceConfig;
codexStart.MetricsEnabled = config.RecordMetrics;
codexStart.MetricsMode = config.Metrics;
if (config.BlockTTL != Configuration.SecondsIn1Day)
{

5
CodexNetDeployer/Configuration.cs
View File

@ -1,5 +1,6 @@
using ArgsUniform;
using DistTestCore.Codex;
using DistTestCore.Metrics;
namespace CodexNetDeployer
{
@ -43,8 +44,8 @@ namespace CodexNetDeployer
[Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
public int BlockTTL { get; set; } = SecondsIn1Day;
[Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
public bool RecordMetrics { get; set; } = false;
[Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
public MetricsMode Metrics { get; set; } = MetricsMode.None;
[Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
"set this option to override the label value.")]

18
CodexNetDeployer/Deployer.cs
View File

@ -27,7 +27,7 @@ namespace CodexNetDeployer
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
var setup = new CodexSetup(999, config.CodexLogLevel);
setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
setup.MetricsEnabled = config.RecordMetrics;
setup.MetricsMode = config.Metrics;
Log("Creating Geth instance and deploying contracts...");
var gethStarter = new GethStarter(lifecycle);
@ -52,9 +52,9 @@ namespace CodexNetDeployer
if (container != null) codexContainers.Add(container);
}
var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
}
private TestLifecycle CreateTestLifecycle()
@ -74,13 +74,19 @@ namespace CodexNetDeployer
return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
}
private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
{
if (!setup.MetricsEnabled) return null;
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
Log("Starting metrics service...");
var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
Log("Starting dashboard service...");
var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
return (prometheusContainer, grafanaStartInfo);
}
private string? GetKubeConfig(string kubeConfigFile)

3
CodexNetDeployer/Program.cs
View File

@ -30,7 +30,8 @@ public class Program
$"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
$"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
$"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
$"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
$"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
$"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
if (!args.Any(a => a == "-y"))
{

2
CodexNetDeployer/deploy-continuous-testnet.sh
View File

@ -10,4 +10,4 @@ dotnet run \
--max-collateral=1024 \
--max-duration=3600000 \
--block-ttl=300 \
--record-metrics=true
--metrics=Dashboard

10
ContinuousTests/ContinuousLogDownloader.cs
View File

@ -7,14 +7,14 @@ namespace ContinuousTests
public class ContinuousLogDownloader
{
private readonly TestLifecycle lifecycle;
private readonly CodexDeployment deployment;
private readonly RunningContainer[] containers;
private readonly string outputPath;
private readonly CancellationToken cancelToken;
public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
public ContinuousLogDownloader(TestLifecycle lifecycle, RunningContainer[] containers, string outputPath, CancellationToken cancelToken)
{
this.lifecycle = lifecycle;
this.deployment = deployment;
this.containers = containers;
this.outputPath = outputPath;
this.cancelToken = cancelToken;
}
@ -37,7 +37,7 @@ namespace ContinuousTests
private void UpdateLogs()
{
foreach (var container in deployment.CodexContainers)
foreach (var container in containers)
{
UpdateLog(container);
}
@ -53,7 +53,7 @@ namespace ContinuousTests
var appender = new LogAppender(filepath);
lifecycle.CodexStarter.DownloadLog(container, appender);
lifecycle.CodexStarter.DownloadLog(container, appender, null);
}
private static string GetLogName(RunningContainer container)

5
ContinuousTests/ContinuousTestRunner.cs
View File

@ -30,7 +30,8 @@ namespace ContinuousTests
ClearAllCustomNamespaces(allTests, overviewLog);
StartLogDownloader(taskFactory);
// Disabled for now. Still looking for a better way.
// StartLogDownloader(taskFactory);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
@ -72,7 +73,7 @@ namespace ContinuousTests
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment.CodexContainers, path, cancelToken);
taskFactory.Run(downloader.Run);
}

16
ContinuousTests/NodeRunner.cs
View File

@ -5,6 +5,7 @@ using KubernetesWorkflow;
using NUnit.Framework;
using Logging;
using Utils;
using DistTestCore.Logs;
namespace ContinuousTests
{
@ -38,6 +39,21 @@ namespace ContinuousTests
RunNode(bootstrapNode, operation, 0.TestTokens());
}
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var subFile = log.CreateSubfile();
var description = container.Name;
var handler = new LogDownloadHandler(container, description, subFile);
log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
var lifecycle = CreateTestLifecycle();
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
flow.DownloadContainerLog(container, handler, tailLines);
return new DownloadedLog(subFile, description);
}
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
{
var lifecycle = CreateTestLifecycle();

103
ContinuousTests/Tests/HoldMyBeerTest.cs Normal file
View File

@ -0,0 +1,103 @@
using DistTestCore;
using Logging;
using NUnit.Framework;
namespace ContinuousTests.Tests
{
public class HoldMyBeerTest : ContinuousTest
{
public override int RequiredNumberOfNodes => 1;
public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
private ContentId? cid;
private TestFile file = null!;
[TestMoment(t: Zero)]
public void UploadTestFile()
{
var filesize = 80.MB();
double codexDefaultBlockSize = 31 * 64 * 33;
var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
var sizeInBytes = filesize.SizeInBytes;
Assert.That(numberOfBlocks, Is.EqualTo(1282));
file = FileManager.GenerateTestFile(filesize);
cid = UploadFile(Nodes[0], file);
Assert.That(cid, Is.Not.Null);
var cidTag = cid!.Id.Substring(cid.Id.Length - 6);
Stopwatch.Measure(Log, "upload-log-asserts", () =>
{
var uploadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000);
var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
uploadLog.DeleteFile();
var storeLine = GetLineForCidTag(storeLines, cidTag);
AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
});
var dl = DownloadFile(Nodes[0], cid!);
file.AssertIsEqual(dl);
Stopwatch.Measure(Log, "download-log-asserts", () =>
{
var downloadLog = NodeRunner.DownloadLog(Nodes[0].Container, 50000);
var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
downloadLog.DeleteFile();
var sentLine = GetLineForCidTag(sentLines, cidTag);
AssertSentLineContains(sentLine, sizeInBytes);
});
}
private void AssertSentLineContains(string sentLine, long sizeInBytes)
{
var tag = "bytes=";
var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
var bytes = Convert.ToInt64(token);
Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
}
private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
{
var tokens = storeLine.Split(" ");
var blocksToken = GetToken(tokens, "blocks=");
var sizeToken = GetToken(tokens, "size=");
if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
var blocks = Convert.ToInt64(blocksToken);
var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
var lineLog = $" Line: '{storeLine}'";
Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
}
private string GetLineForCidTag(string[] lines, string cidTag)
{
var result = lines.SingleOrDefault(l => l.Contains(cidTag));
if (result == null)
{
Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
throw new Exception();
}
return result;
}
private string? GetToken(string[] tokens, string tag)
{
var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
if (token == null) return null;
return token.Substring(tag.Length);
}
}
}

10
DistTestCore/ByteSize.cs
View File

@ -38,6 +38,11 @@ namespace DistTestCore
{
private const long Kilo = 1024;
public static ByteSize Bytes(this long i)
{
return new ByteSize(i);
}
public static ByteSize KB(this long i)
{
return new ByteSize(i * Kilo);
@ -58,6 +63,11 @@ namespace DistTestCore
return (i * Kilo).GB();
}
public static ByteSize Bytes(this int i)
{
return new ByteSize(i);
}
public static ByteSize KB(this int i)
{
return Convert.ToInt64(i).KB();

29
DistTestCore/Codex/CodexAccess.cs
View File

@ -4,10 +4,11 @@ using Utils;
namespace DistTestCore.Codex
{
public class CodexAccess
public class CodexAccess : ILogHandler
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private bool hasContainerCrashed;
public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
{
@ -15,6 +16,9 @@ namespace DistTestCore.Codex
Container = container;
this.timeSet = timeSet;
Address = address;
hasContainerCrashed = false;
if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
}
public RunningContainer Container { get; }
@ -86,7 +90,30 @@ namespace DistTestCore.Codex
private Http Http()
{
CheckContainerCrashed();
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
}
private void CheckContainerCrashed()
{
if (hasContainerCrashed) throw new Exception("Container has crashed.");
}
public void Log(Stream crashLog)
{
var file = log.CreateSubfile();
log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
using var reader = new StreamReader(crashLog);
var line = reader.ReadLine();
while (line != null)
{
file.Write(line);
line = reader.ReadLine();
}
log.Log("Crash log successfully downloaded.");
hasContainerCrashed = true;
}
}
}

10
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -51,7 +51,15 @@ namespace DistTestCore.Codex
{
AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
}
if (config.MetricsEnabled)
if (config.BlockMaintenanceInterval != null)
{
AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
}
if (config.BlockMaintenanceNumber != null)
{
AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
}
if (config.MetricsMode != Metrics.MetricsMode.None)
{
AddEnvVar("CODEX_METRICS", "true");
AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");

4
DistTestCore/Codex/CodexDeployment.cs
View File

@ -5,17 +5,19 @@ namespace DistTestCore.Codex
{
public class CodexDeployment
{
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
{
GethStartResult = gethStartResult;
CodexContainers = codexContainers;
PrometheusContainer = prometheusContainer;
GrafanaStartInfo = grafanaStartInfo;
Metadata = metadata;
}
public GethStartResult GethStartResult { get; }
public RunningContainer[] CodexContainers { get; }
public RunningContainer? PrometheusContainer { get; }
public GrafanaStartInfo? GrafanaStartInfo { get; }
public DeploymentMetadata Metadata { get; }
}

5
DistTestCore/Codex/CodexStartupConfig.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using KubernetesWorkflow;
namespace DistTestCore.Codex
@ -14,9 +15,11 @@ namespace DistTestCore.Codex
public Location Location { get; set; }
public CodexLogLevel LogLevel { get; }
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MetricsMode MetricsMode { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
public string? BootstrapSpr { get; set; }
public int? BlockTTL { get; set; }
public TimeSpan? BlockMaintenanceInterval { get; set; }
public int? BlockMaintenanceNumber { get; set; }
}
}

16
DistTestCore/CodexSetup.cs
View File

@ -11,6 +11,8 @@ namespace DistTestCore
ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup WithBlockTTL(TimeSpan duration);
ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
ICodexSetup EnableMetrics();
ICodexSetup EnableMarketplace(TestToken initialBalance);
ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@ -57,9 +59,21 @@ namespace DistTestCore
return this;
}
public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
{
BlockMaintenanceInterval = duration;
return this;
}
public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
{
BlockMaintenanceNumber = numberOfBlocks;
return this;
}
public ICodexSetup EnableMetrics()
{
MetricsEnabled = true;
MetricsMode = Metrics.MetricsMode.Record;
return this;
}

36
DistTestCore/CodexStarter.cs
View File

@ -47,7 +47,11 @@ namespace DistTestCore
{
LogStart($"Stopping {group.Describe()}...");
var workflow = CreateWorkflow();
foreach (var c in group.Containers) workflow.Stop(c);
foreach (var c in group.Containers)
{
StopCrashWatcher(c);
workflow.Stop(c);
}
RunningGroups.Remove(group);
LogEnd("Stopped.");
}
@ -60,17 +64,23 @@ namespace DistTestCore
RunningGroups.Clear();
}
public void DownloadLog(RunningContainer container, ILogHandler logHandler)
public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
{
var workflow = CreateWorkflow();
workflow.DownloadContainerLog(container, logHandler);
workflow.DownloadContainerLog(container, logHandler, tailLines);
}
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
{
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
if (codexSetup.MetricsMode == MetricsMode.Dashboard)
{
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
}
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
}
@ -90,7 +100,9 @@ namespace DistTestCore
for (var i = 0; i < numberOfNodes; i++)
{
var workflow = CreateWorkflow();
result.Add(workflow.Start(1, location, recipe, startupConfig));
var rc = workflow.Start(1, location, recipe, startupConfig);
CreateCrashWatcher(workflow, rc);
result.Add(rc);
}
return result.ToArray();
}
@ -128,5 +140,19 @@ namespace DistTestCore
{
Log("----------------------------------------------------------------------------");
}
private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
{
var c = rc.Containers.Single();
c.CrashWatcher = workflow.CreateCrashWatcher(c);
}
private void StopCrashWatcher(RunningContainers containers)
{
foreach (var c in containers.Containers)
{
c.CrashWatcher?.Stop();
}
}
}
}

5
DistTestCore/DistTest.cs
View File

@ -175,6 +175,11 @@ namespace DistTestCore
GetTestLog().Debug(msg);
}
public void Measure(string name, Action action)
{
Stopwatch.Measure(Get().Log, name, action);
}
protected CodexSetup CreateCodexSetup(int numberOfNodes)
{
return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());

8
DistTestCore/DistTestCore.csproj
View File

@ -10,6 +10,14 @@
<PropertyGroup Condition="'$(IsArm64)'=='true'">
<DefineConstants>Arm64</DefineConstants>
</PropertyGroup>
<ItemGroup>
<None Remove="Metrics\dashboard.json" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Metrics\dashboard.json">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

192
DistTestCore/GrafanaStarter.cs Normal file
View File

@ -0,0 +1,192 @@
using DistTestCore.Metrics;
using IdentityModel.Client;
using KubernetesWorkflow;
using Newtonsoft.Json;
using System.Reflection;
namespace DistTestCore
{
public class GrafanaStarter : BaseStarter
{
private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
public GrafanaStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
{
LogStart($"Starting dashboard server");
var grafanaContainer = StartGrafanaContainer();
var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
Log("Connecting datasource...");
AddDataSource(http, prometheusContainer);
Log("Uploading dashboard configurations...");
var jsons = ReadEachDashboardJsonFile(codexSetup);
var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
LogEnd("Dashboard server started.");
return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
}
private RunningContainer StartGrafanaContainer()
{
var startupConfig = new StartupConfig();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
return grafanaContainers.Containers.First();
}
private void AddBasicAuth(HttpClient client)
{
client.SetBasicAuthentication(
GrafanaContainerRecipe.DefaultAdminUser,
GrafanaContainerRecipe.DefaultAdminPassword);
}
private static void AddDataSource(Http http, RunningContainer prometheusContainer)
{
var prometheusAddress = prometheusContainer.ClusterExternalAddress;
var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
{
uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
name = "CodexPrometheus",
type = "prometheus",
url = prometheusUrl,
access = "proxy",
basicAuth = false,
jsonData = new GrafanaDataSourceJsonData
{
httpMethod = "POST"
}
});
if (response.message != "Datasource added")
{
throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
}
}
public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
{
var request = GetDashboardCreateRequest(dashboardJson);
var response = http.HttpPostString("dashboards/db", request);
var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
var grafanaAddress = grafanaContainer.ClusterExternalAddress;
return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
}
private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceNames = new[]
{
"DistTestCore.Metrics.dashboard.json"
};
return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
}
private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
{
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream == null) throw new Exception("Unable to find resource " + resourceName);
using var reader = new StreamReader(stream);
return ApplyReplacements(reader.ReadToEnd(), codexSetup);
}
private static string ApplyReplacements(string input, CodexSetup codexSetup)
{
var quotaString = GetQuotaString(codexSetup);
var softMaxString = GetSoftMaxString(codexSetup);
return input
.Replace(StorageQuotaThresholdReplaceToken, quotaString)
.Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
}
private static string GetQuotaString(CodexSetup codexSetup)
{
return GetCodexStorageQuotaInBytes(codexSetup).ToString();
}
private static string GetSoftMaxString(CodexSetup codexSetup)
{
var quota = GetCodexStorageQuotaInBytes(codexSetup);
var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
return softMax.ToString();
}
private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
{
if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
// Codex default: 8GB
return 8.GB().SizeInBytes;
}
private static string GetDashboardCreateRequest(string dashboardJson)
{
return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
}
}
public class GrafanaStartInfo
{
public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
{
DashboardUrls = dashboardUrls;
Container = container;
}
public string[] DashboardUrls { get; }
public RunningContainer Container { get; }
}
public class GrafanaDataSourceRequest
{
public string uid { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
public string type { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public string access { get; set; } = string.Empty;
public bool basicAuth { get; set; }
public GrafanaDataSourceJsonData jsonData { get; set; } = new();
}
public class GrafanaDataSourceResponse
{
public int id { get; set; }
public string message { get; set; } = string.Empty;
public string name { get; set; } = string.Empty;
}
public class GrafanaDataSourceJsonData
{
public string httpMethod { get; set; } = string.Empty;
}
public class GrafanaPostDashboardResponse
{
public int id { get; set; }
public string slug { get; set; } = string.Empty;
public string status { get; set; } = string.Empty;
public string uid { get; set; } = string.Empty;
public string url { get; set; } = string.Empty;
public int version { get; set; }
}
}

30
DistTestCore/Http.cs
View File

@ -12,14 +12,21 @@ namespace DistTestCore
private readonly ITimeSet timeSet;
private readonly Address address;
private readonly string baseUrl;
private readonly Action<HttpClient> onClientCreated;
private readonly string? logAlias;
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
: this(log, timeSet, address, baseUrl, DoNothing, logAlias)
{
}
public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
{
this.log = log;
this.timeSet = timeSet;
this.address = address;
this.baseUrl = baseUrl;
this.onClientCreated = onClientCreated;
this.logAlias = logAlias;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@ -66,6 +73,22 @@ namespace DistTestCore
}, $"HTTP-POST-JSON: {route}");
}
public string HttpPostString(string route, string body)
{
return Retry(() =>
{
using var client = GetClient();
var url = GetUrl() + route;
Log(url, body);
var content = new StringContent(body);
content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
var result = Time.Wait(client.PostAsync(url, content));
var str = Time.Wait(result.Content.ReadAsStringAsync());
Log(url, str);
return str;
}, $"HTTP-POST-STRING: {route}");
}
public string HttpPostStream(string route, Stream stream)
{
return Retry(() =>
@ -76,7 +99,7 @@ namespace DistTestCore
var content = new StreamContent(stream);
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = Time.Wait(client.PostAsync(url, content));
var str =Time.Wait(response.Content.ReadAsStringAsync());
var str = Time.Wait(response.Content.ReadAsStringAsync());
Log(url, str);
return str;
}, $"HTTP-POST-STREAM: {route}");
@ -132,7 +155,12 @@ namespace DistTestCore
{
var client = new HttpClient();
client.Timeout = timeSet.HttpCallTimeout();
onClientCreated(client);
return client;
}
private static void DoNothing(HttpClient client)
{
}
}
}

27
DistTestCore/Logs/DownloadedLog.cs
View File

@ -6,6 +6,8 @@ namespace DistTestCore.Logs
public interface IDownloadedLog
{
void AssertLogContains(string expectedString);
string[] FindLinesThatContain(params string[] tags);
void DeleteFile();
}
public class DownloadedLog : IDownloadedLog
@ -33,5 +35,30 @@ namespace DistTestCore.Logs
Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
}
public string[] FindLinesThatContain(params string[] tags)
{
var result = new List<string>();
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
while (line != null)
{
if (tags.All(line.Contains))
{
result.Add(line);
}
line = streamReader.ReadLine();
}
return result.ToArray();
}
public void DeleteFile()
{
File.Delete(logFile.FullFilename);
}
}
}

2
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
WaitUntil(() =>
{
var logHandler = new ContractsReadyLogHandler(Debug);
workflow.DownloadContainerLog(container, logHandler);
workflow.DownloadContainerLog(container, logHandler, null);
return logHandler.Found;
});
Log("Contracts deployed. Extracting addresses...");

2
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
private string FetchPubKey()
{
var enodeFinder = new PubKeyFinder(s => log.Debug(s));
workflow.DownloadContainerLog(container, enodeFinder);
workflow.DownloadContainerLog(container, enodeFinder, null);
return enodeFinder.GetPubKey();
}

25
DistTestCore/Metrics/GrafanaContainerRecipe.cs Normal file
View File

@ -0,0 +1,25 @@
using KubernetesWorkflow;
namespace DistTestCore.Metrics
{
public class GrafanaContainerRecipe : ContainerRecipeFactory
{
public override string AppName => "grafana";
public override string Image => "grafana/grafana-oss:10.0.3";
public const string DefaultAdminUser = "adminium";
public const string DefaultAdminPassword = "passwordium";
protected override void Initialize(StartupConfig startupConfig)
{
AddExposedPort(3000);
AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
}
}
}

9
DistTestCore/Metrics/MetricsMode.cs Normal file
View File

@ -0,0 +1,9 @@
namespace DistTestCore.Metrics
{
public enum MetricsMode
{
None,
Record,
Dashboard
}
}

1847
DistTestCore/Metrics/dashboard.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
DistTestCore/OnlineCodexNode.cs
View File

@ -4,6 +4,7 @@ using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using Logging;
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -15,7 +16,7 @@ namespace DistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
void ConnectToPeer(IOnlineCodexNode node);
IDownloadedLog DownloadLog();
IDownloadedLog DownloadLog(int? tailLines = null);
IMetricsAccess Metrics { get; }
IMarketplaceAccess Marketplace { get; }
CodexDebugVersionResponse Version { get; }
@ -67,6 +68,7 @@ namespace DistTestCore
using var fileStream = File.OpenRead(file.Filename);
var logMessage = $"Uploading file {file.Describe()}...";
Log(logMessage);
var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
{
return CodexAccess.UploadFile(fileStream);
@ -82,6 +84,7 @@ namespace DistTestCore
public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
Log(logMessage);
var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@ -100,9 +103,9 @@ namespace DistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public IDownloadedLog DownloadLog()
public IDownloadedLog DownloadLog(int? tailLines = null)
{
return lifecycle.DownloadLog(CodexAccess.Container);
return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
}
public ICodexSetup BringOffline()
@ -116,7 +119,7 @@ namespace DistTestCore
public void EnsureOnlineGetVersionResponse()
{
var debugInfo = CodexAccess.GetDebugInfo();
var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
var nodePeerId = debugInfo.id;
var nodeName = CodexAccess.Container.Name;

4
DistTestCore/PrometheusStarter.cs
View File

@ -22,8 +22,6 @@ namespace DistTestCore
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
LogEnd("Metrics server started.");
return runningContainers;
}
@ -31,7 +29,7 @@ namespace DistTestCore
{
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_interval: 10s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";

9
DistTestCore/TestLifecycle.cs
View File

@ -24,6 +24,7 @@ namespace DistTestCore
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this);
PrometheusStarter = new PrometheusStarter(this);
GrafanaStarter = new GrafanaStarter(this);
GethStarter = new GethStarter(this);
testStart = DateTime.UtcNow;
CodexVersion = null;
@ -38,6 +39,7 @@ namespace DistTestCore
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
public GrafanaStarter GrafanaStarter { get; }
public GethStarter GethStarter { get; }
public CodexDebugVersionResponse? CodexVersion { get; private set; }
@ -47,14 +49,14 @@ namespace DistTestCore
FileManager.DeleteAllTestFiles();
}
public IDownloadedLog DownloadLog(RunningContainer container)
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var subFile = Log.CreateSubfile();
var description = container.Name;
var handler = new LogDownloadHandler(container, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
CodexStarter.DownloadLog(container, handler);
CodexStarter.DownloadLog(container, handler, tailLines);
return new DownloadedLog(subFile, description);
}
@ -76,7 +78,8 @@ namespace DistTestCore
codexId: GetCodexId(),
gethId: new GethContainerRecipe().Image,
prometheusId: new PrometheusContainerRecipe().Image,
codexContractsId: new CodexContractsContainerRecipe().Image
codexContractsId: new CodexContractsContainerRecipe().Image,
grafanaId: new GrafanaContainerRecipe().Image
);
}

6
DistTestCore/Timing.cs
View File

@ -21,7 +21,7 @@ namespace DistTestCore
{
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromSeconds(10);
return TimeSpan.FromMinutes(5);
}
public TimeSpan HttpCallRetryTime()
@ -36,12 +36,12 @@ namespace DistTestCore
public TimeSpan WaitForK8sServiceDelay()
{
return TimeSpan.FromSeconds(1);
return TimeSpan.FromSeconds(10);
}
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(1);
return TimeSpan.FromMinutes(30);
}
public TimeSpan WaitForMetricTimeout()

7
KubernetesWorkflow/ContainerRecipeFactory.cs
View File

@ -40,6 +40,13 @@
return p;
}
protected Port AddExposedPort(int number, string tag = "")
{
var p = factory.CreatePort(number, tag);
exposedPorts.Add(p);
return p;
}
protected Port AddInternalPort(string tag = "")
{
var p = factory.CreatePort(tag);

85
KubernetesWorkflow/CrashWatcher.cs Normal file
View File

@ -0,0 +1,85 @@
using k8s;
using Logging;
namespace KubernetesWorkflow
{
public class CrashWatcher
{
private readonly BaseLog log;
private readonly KubernetesClientConfiguration config;
private readonly string k8sNamespace;
private readonly RunningContainer container;
private ILogHandler? logHandler;
private CancellationTokenSource cts;
private Task? worker;
private Exception? workerException;
public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
{
this.log = log;
this.config = config;
this.k8sNamespace = k8sNamespace;
this.container = container;
cts = new CancellationTokenSource();
}
public void Start(ILogHandler logHandler)
{
if (worker != null) throw new InvalidOperationException();
this.logHandler = logHandler;
cts = new CancellationTokenSource();
worker = Task.Run(Worker);
}
public void Stop()
{
if (worker == null) throw new InvalidOperationException();
cts.Cancel();
worker.Wait();
worker = null;
if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
}
private void Worker()
{
try
{
MonitorContainer(cts.Token);
}
catch (Exception ex)
{
workerException = ex;
}
}
private void MonitorContainer(CancellationToken token)
{
var client = new Kubernetes(config);
while (!token.IsCancellationRequested)
{
token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
var pod = container.Pod;
var recipe = container.Recipe;
var podName = pod.PodInfo.Name;
var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
{
DownloadCrashedContainerLogs(client, podName, recipe);
return;
}
}
}
private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
{
log.Log("Pod crash detected for " + container.Name);
using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
logHandler!.Log(stream);
}
}
}

9
KubernetesWorkflow/K8sController.cs
View File

@ -53,10 +53,10 @@ namespace KubernetesWorkflow
WaitUntilPodOffline(pod.PodInfo.Name);
}
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
{
log.Debug();
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
logHandler.Log(stream);
}
@ -604,6 +604,11 @@ namespace KubernetesWorkflow
#endregion
public CrashWatcher CreateCrashWatcher(RunningContainer container)
{
return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
}
private PodInfo FetchNewPod()
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

1
KubernetesWorkflow/PodLabels.cs
View File

@ -25,6 +25,7 @@ namespace KubernetesWorkflow
Add("gethid", applicationIds.GethId);
Add("prometheusid", applicationIds.PrometheusId);
Add("codexcontractsid", applicationIds.CodexContractsId);
Add("grafanaid", applicationIds.GrafanaId);
}
public PodLabels GetLabelsForAppName(string appName)

5
KubernetesWorkflow/RecipeComponentFactory.cs
View File

@ -7,6 +7,11 @@ namespace KubernetesWorkflow
{
private NumberSource portNumberSource = new NumberSource(8080);
public Port CreatePort(int number, string tag)
{
return new Port(number, tag);
}
public Port CreatePort(string tag)
{
return new Port(portNumberSource.GetNextNumber(), tag);

6
KubernetesWorkflow/RunningContainers.cs
View File

@ -1,4 +1,5 @@
using Utils;
using Newtonsoft.Json;
using Utils;
namespace KubernetesWorkflow
{
@ -39,6 +40,9 @@ namespace KubernetesWorkflow
public Port[] ServicePorts { get; }
public Address ClusterExternalAddress { get; }
public Address ClusterInternalAddress { get; }
[JsonIgnore]
public CrashWatcher? CrashWatcher { get; set; }
}
public static class RunningContainersExtensions

9
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -37,6 +37,11 @@ namespace KubernetesWorkflow
}, pl);
}
public CrashWatcher CreateCrashWatcher(RunningContainer container)
{
return K8s(controller => controller.CreateCrashWatcher(container));
}
public void Stop(RunningContainers runningContainers)
{
K8s(controller =>
@ -45,11 +50,11 @@ namespace KubernetesWorkflow
});
}
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
{
K8s(controller =>
{
controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
});
}

4
Logging/ApplicationIds.cs
View File

@ -2,17 +2,19 @@
{
public class ApplicationIds
{
public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
{
CodexId = codexId;
GethId = gethId;
PrometheusId = prometheusId;
CodexContractsId = codexContractsId;
GrafanaId = grafanaId;
}
public string CodexId { get; }
public string GethId { get; }
public string PrometheusId { get; }
public string CodexContractsId { get; }
public string GrafanaId { get; }
}
}

2
Logging/StatusLog.cs
View File

@ -26,6 +26,7 @@ namespace Logging
gethid = applicationIds.GethId,
prometheusid = applicationIds.PrometheusId,
codexcontractsid = applicationIds.CodexContractsId,
grafanaid = applicationIds.GrafanaId,
category = NameUtils.GetCategoryName(),
fixturename = fixtureName,
testname = NameUtils.GetTestMethodName(),
@ -59,6 +60,7 @@ namespace Logging
public string gethid { get; set; } = string.Empty;
public string prometheusid { get; set; } = string.Empty;
public string codexcontractsid { get; set; } = string.Empty;
public string grafanaid { get; set; } = string.Empty;
public string category { get; set; } = string.Empty;
public string fixturename { get; set; } = string.Empty;
public string testname { get; set; } = string.Empty;

194
Tests/BasicTests/ContinuousSubstitute.cs Normal file
View File

@ -0,0 +1,194 @@
using DistTestCore;
using Logging;
using NUnit.Framework;
using Utils;
namespace Tests.BasicTests
{
[TestFixture]
public class ContinuousSubstitute : AutoBootstrapDistTest
{
[Test]
[UseLongTimeouts]
public void ContinuousTestSubstitute()
{
var group = SetupCodexNodes(5, o => o
.EnableMetrics()
.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
.WithBlockTTL(TimeSpan.FromMinutes(2))
.WithStorageQuota(3.GB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray();
foreach (var node in nodes)
{
node.Marketplace.MakeStorageAvailable(
size: 1.GB(),
minPricePerBytePerSecond: 1.TestTokens(),
maxCollateral: 1024.TestTokens(),
maxDuration: TimeSpan.FromMinutes(5));
}
var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
while (DateTime.UtcNow < endTime)
{
var allNodes = nodes.ToList();
var primary = allNodes.PickOneRandom();
var secondary = allNodes.PickOneRandom();
Log("Run Test");
PerformTest(primary, secondary);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
private ByteSize fileSize = 80.MB();
private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{
ScopedTestFiles(() =>
{
var testFile = GenerateTestFile(fileSize);
var contentId = primary.UploadFile(testFile);
var downloadedFile = secondary.DownloadContent(contentId);
testFile.AssertIsEqual(downloadedFile);
});
}
[Test]
public void HoldMyBeerTest()
{
var blockExpirationTime = TimeSpan.FromMinutes(3);
var group = SetupCodexNodes(3, o => o
.EnableMetrics()
.WithBlockTTL(blockExpirationTime)
.WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
.WithBlockMaintenanceNumber(10000)
.WithStorageQuota(2000.MB()));
var nodes = group.Cast<OnlineCodexNode>().ToArray();
var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
var filesize = 80.MB();
double codexDefaultBlockSize = 31 * 64 * 33;
var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
var sizeInBytes = filesize.SizeInBytes;
Assert.That(numberOfBlocks, Is.EqualTo(1282));
var startTime = DateTime.UtcNow;
var successfulUploads = 0;
var successfulDownloads = 0;
while (DateTime.UtcNow < endTime)
{
foreach (var node in nodes)
{
try
{
Thread.Sleep(TimeSpan.FromSeconds(5));
ScopedTestFiles(() =>
{
var uploadStartTime = DateTime.UtcNow;
var file = GenerateTestFile(filesize);
var cid = node.UploadFile(file);
var cidTag = cid.Id.Substring(cid.Id.Length - 6);
Measure("upload-log-asserts", () =>
{
var uploadLog = node.DownloadLog(tailLines: 50000);
var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
uploadLog.DeleteFile();
var storeLine = GetLineForCidTag(storeLines, cidTag);
AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
});
successfulUploads++;
var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
{
Assert.Fail("Upload took too long. Blocks already expired.");
}
var dl = node.DownloadContent(cid);
file.AssertIsEqual(dl);
Measure("download-log-asserts", () =>
{
var downloadLog = node.DownloadLog(tailLines: 50000);
var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
downloadLog.DeleteFile();
var sentLine = GetLineForCidTag(sentLines, cidTag);
AssertSentLineContains(sentLine, sizeInBytes);
});
successfulDownloads++;
});
}
catch
{
var testDuration = DateTime.UtcNow - startTime;
Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
Thread.Sleep(TimeSpan.FromSeconds(30));
throw;
}
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
private void AssertSentLineContains(string sentLine, long sizeInBytes)
{
var tag = "bytes=";
var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
var bytes = Convert.ToInt64(token);
Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
}
private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
{
var tokens = storeLine.Split(" ");
var blocksToken = GetToken(tokens, "blocks=");
var sizeToken = GetToken(tokens, "size=");
if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
var blocks = Convert.ToInt64(blocksToken);
var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
var lineLog = $" Line: '{storeLine}'";
Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
}
private string GetLineForCidTag(string[] lines, string cidTag)
{
var result = lines.SingleOrDefault(l => l.Contains(cidTag));
if (result == null)
{
Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
throw new Exception();
}
return result;
}
private string? GetToken(string[] tokens, string tag)
{
var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
if (token == null) return null;
return token.Substring(tag.Length);
}
}
}

1
Tests/Tests.csproj
View File

@ -13,6 +13,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
<ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
</ItemGroup>