status-im/cs-codex-dist-tests
2
0
Fork 0
mirror of synced 2025-02-04 20:54:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/peer-connection-check-on-deploy'

Browse Source
This commit is contained in:
benbierens 2023-08-29 14:48:14 +02:00
commit 6bb7b994ed
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
19 changed files with 434 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -21,7 +21,7 @@ namespace CodexNetDeployer
validatorsLeft = numberOfValidators;
}
public RunningContainer? Start(int i)
public CodexNodeStartResult? Start(int i)
{
Console.Write($" - {i} = ");
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
@ -60,7 +60,7 @@ namespace CodexNetDeployer
if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
validatorsLeft--;
return container;
return new CodexNodeStartResult(workflow, container, codexAccess);
}
}
}
@ -102,4 +102,18 @@ namespace CodexNetDeployer
return codexStart;
}
}
public class CodexNodeStartResult
{
public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
{
Workflow = workflow;
Container = container;
Access = access;
}
public StartupWorkflow Workflow { get; }
public RunningContainer Container { get; }
public CodexAccess Access { get; }
}
}

3
CodexNetDeployer/Configuration.cs
View File

@ -57,6 +57,9 @@ namespace CodexNetDeployer
[Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
"set this option to override the label value.")]
public string TestsTypePodLabel { get; set; } = "continuous-tests";
[Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
public bool CheckPeerConnection { get; set; } = false;
public List<string> Validate()
{

50
CodexNetDeployer/Deployer.cs
View File

@ -8,14 +8,14 @@ namespace CodexNetDeployer
public class Deployer
{
private readonly Configuration config;
private readonly NullLog log;
private readonly DefaultTimeSet timeset;
private readonly PeerConnectivityChecker peerConnectivityChecker;
public Deployer(Configuration config)
{
this.config = config;
log = new NullLog();
timeset = new DefaultTimeSet();
peerConnectivityChecker = new PeerConnectivityChecker();
}
public CodexDeployment Deploy()
@ -45,16 +45,19 @@ namespace CodexNetDeployer
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexContainers = new List<RunningContainer>();
var startResults = new List<CodexNodeStartResult>();
for (var i = 0; i < config.NumberOfCodexNodes; i++)
{
var container = codexStarter.Start(i);
if (container != null) codexContainers.Add(container);
var result = codexStarter.Start(i);
if (result != null) startResults.Add(result);
}
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
CheckPeerConnectivity(startResults);
CheckContainerRestarts(startResults);
return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
}
private TestLifecycle CreateTestLifecycle()
@ -71,10 +74,10 @@ namespace CodexNetDeployer
k8sNamespacePrefix: config.KubeNamespace
);
return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
}
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
{
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
@ -95,6 +98,35 @@ namespace CodexNetDeployer
return kubeConfigFile;
}
private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
{
if (!config.CheckPeerConnection) return;
Log("Starting peer-connectivity check for deployed nodes...");
peerConnectivityChecker.CheckConnectivity(codexContainers);
Log("Check passed.");
}
private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
{
var crashes = new List<RunningContainer>();
foreach (var startResult in startResults)
{
var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
}
if (!crashes.Any())
{
Log("Container restart check passed.");
}
else
{
Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
throw new Exception("Deployment failed: One or more containers crashed.");
}
}
private DeploymentMetadata CreateMetadata()
{
return new DeploymentMetadata(

34
CodexNetDeployer/PeerConnectivityChecker.cs Normal file
View File

@ -0,0 +1,34 @@
using DistTestCore.Helpers;
using Logging;
namespace CodexNetDeployer
{
public class PeerConnectivityChecker
{
public void CheckConnectivity(List<CodexNodeStartResult> startResults)
{
var log = new ConsoleLog();
var checker = new PeerConnectionTestHelpers(log);
var access = startResults.Select(r => r.Access);
checker.AssertFullyConnected(access);
}
}
public class ConsoleLog : BaseLog
{
public ConsoleLog() : base(false)
{
}
protected override string GetFullName()
{
return "CONSOLE";
}
public override void Log(string message)
{
Console.WriteLine(message);
}
}
}

3
CodexNetDeployer/Program.cs
View File

@ -56,8 +56,5 @@ public class Program
Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
"The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
"All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
"If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
}
}

3
CodexNetDeployer/deploy-continuous-testnet.sh
View File

@ -12,4 +12,5 @@ dotnet run \
--block-ttl=180 \
--block-mi=120 \
--block-mn=10000 \
--metrics=Dashboard
--metrics=Dashboard \
--check-connect=1

69
ContinuousTests/Tests/HoldMyBeerTest.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using Logging;
using NUnit.Framework;
namespace ContinuousTests.Tests
@ -26,7 +25,6 @@ namespace ContinuousTests.Tests
var filesize = filesizePerUploadMb.MB();
double codexDefaultBlockSize = 31 * 64 * 33;
var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
var sizeInBytes = filesize.SizeInBytes;
Assert.That(numberOfBlocks, Is.EqualTo(1282));
file = FileManager.GenerateTestFile(filesize);
@ -34,75 +32,8 @@ namespace ContinuousTests.Tests
cid = UploadFile(Nodes[0], file);
Assert.That(cid, Is.Not.Null);
var cidTag = cid!.Id.Substring(cid.Id.Length - 6);
Stopwatch.Measure(Log, "upload-log-asserts", () =>
{
var uploadLog = DownloadContainerLog(Nodes[0].Container, 50000);
var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
uploadLog.DeleteFile();
var storeLine = GetLineForCidTag(storeLines, cidTag);
AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
});
var dl = DownloadFile(Nodes[0], cid!);
file.AssertIsEqual(dl);
Stopwatch.Measure(Log, "download-log-asserts", () =>
{
var downloadLog = DownloadContainerLog(Nodes[0].Container, 50000);
var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
downloadLog.DeleteFile();
var sentLine = GetLineForCidTag(sentLines, cidTag);
AssertSentLineContains(sentLine, sizeInBytes);
});
}
private void AssertSentLineContains(string sentLine, long sizeInBytes)
{
var tag = "bytes=";
var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
var bytes = Convert.ToInt64(token);
Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
}
private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
{
var tokens = storeLine.Split(" ");
var blocksToken = GetToken(tokens, "blocks=");
var sizeToken = GetToken(tokens, "size=");
if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
var blocks = Convert.ToInt64(blocksToken);
var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
var lineLog = $" Line: '{storeLine}'";
Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
}
private string GetLineForCidTag(string[] lines, string cidTag)
{
var result = lines.SingleOrDefault(l => l.Contains(cidTag));
if (result == null)
{
Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
throw new Exception();
}
return result;
}
private string? GetToken(string[] tokens, string tag)
{
var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
if (token == null) return null;
return token.Substring(tag.Length);
}
}
}

12
DistTestCore/Codex/CodexAccess.cs
View File

@ -88,13 +88,17 @@ namespace DistTestCore.Codex
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http()
public string GetName()
{
CheckContainerCrashed();
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
return Container.Name;
}
private void CheckContainerCrashed()
private Http Http()
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
}
private void CheckContainerCrashed(HttpClient client)
{
if (hasContainerCrashed) throw new Exception("Container has crashed.");
}

16
DistTestCore/DistTest.cs
View File

@ -30,14 +30,8 @@ namespace DistTestCore
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
statusLog = new StatusLog(logConfig, startTime);
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
}
public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
[OneTimeSetUp]
public void GlobalSetup()
{
@ -175,6 +169,16 @@ namespace DistTestCore
GetTestLog().Debug(msg);
}
public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
{
return new PeerConnectionTestHelpers(GetTestLog());
}
public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
{
return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
}
public void Measure(string name, Action action)
{
Stopwatch.Measure(Get().Log, name, action);

34
DistTestCore/Helpers/FullConnectivityHelper.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using Logging;
using NUnit.Framework;
using Utils;
@ -14,23 +15,23 @@ namespace DistTestCore.Helpers
public class FullConnectivityHelper
{
private static string Nl = Environment.NewLine;
private readonly DistTest test;
private readonly BaseLog log;
private readonly IFullConnectivityImplementation implementation;
public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
{
this.test = test;
this.log = log;
this.implementation = implementation;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
private void AssertFullyConnected(CodexAccess[] nodes)
{
test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
@ -43,19 +44,19 @@ namespace DistTestCore.Helpers
{
var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
test.Log($"Connections failed:{Nl}{pairDetails}");
Log($"Connections failed:{Nl}{pairDetails}");
Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
}
else
{
test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
{
action();
@ -72,7 +73,7 @@ namespace DistTestCore.Helpers
foreach (var pair in selectedPair)
{
test.ScopedTestFiles(pair.Check);
pair.Check();
if (pair.Success)
{
@ -81,10 +82,10 @@ namespace DistTestCore.Helpers
}
}
test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
}
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
private Entry[] CreateEntries(CodexAccess[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
@ -117,15 +118,20 @@ namespace DistTestCore.Helpers
}
}
private void Log(string msg)
{
log.Log(msg);
}
public class Entry
{
public Entry(IOnlineCodexNode node)
public Entry(CodexAccess node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexAccess Node { get; }
public CodexDebugResponse Response { get; }
public override string ToString()

15
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using Logging;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
@ -7,12 +8,17 @@ namespace DistTestCore.Helpers
{
private readonly FullConnectivityHelper helper;
public PeerConnectionTestHelpers(DistTest test)
public PeerConnectionTestHelpers(BaseLog log)
{
helper = new FullConnectivityHelper(test, this);
helper = new FullConnectivityHelper(log, this);
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
}
public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
{
helper.AssertFullyConnected(nodes);
}
@ -57,9 +63,8 @@ namespace DistTestCore.Helpers
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
var ip = peer.Node.Container.Pod.PodInfo.Ip;
var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}

42
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,21 +1,30 @@
using static DistTestCore.Helpers.FullConnectivityHelper;
using DistTestCore.Codex;
using Logging;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers : IFullConnectivityImplementation
{
private readonly FullConnectivityHelper helper;
private readonly DistTest test;
private readonly BaseLog log;
private readonly FileManager fileManager;
private ByteSize testFileSize;
public PeerDownloadTestHelpers(DistTest test)
public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
{
helper = new FullConnectivityHelper(test, this);
helper = new FullConnectivityHelper(log, this);
testFileSize = 1.MB();
this.test = test;
this.log = log;
this.fileManager = fileManager;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
}
public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
{
this.testFileSize = testFileSize;
helper.AssertFullyConnected(nodes);
@ -33,13 +42,15 @@ namespace DistTestCore.Helpers
public PeerConnectionState Check(Entry from, Entry to)
{
fileManager.PushFileSet();
var expectedFile = GenerateTestFile(from.Node, to.Node);
var contentId = from.Node.UploadFile(expectedFile);
using var uploadStream = File.OpenRead(expectedFile.Filename);
var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
try
{
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
expectedFile.AssertIsEqual(downloadedFile);
return PeerConnectionState.Connection;
}
@ -49,16 +60,29 @@ namespace DistTestCore.Helpers
// We consider that as no-connection for the purpose of this test.
return PeerConnectionState.NoConnection;
}
finally
{
fileManager.PopFileSet();
}
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
private TestFile DownloadFile(CodexAccess node, string contentId, string label)
{
var downloadedFile = fileManager.CreateEmptyTestFile(label);
using var downloadStream = File.OpenWrite(downloadedFile.Filename);
using var stream = node.DownloadFile(contentId);
stream.CopyTo(downloadStream);
return downloadedFile;
}
private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
{
var up = uploader.GetName().Replace("<", "").Replace(">", "");
var down = downloader.GetName().Replace("<", "").Replace(">", "");
var label = $"~from:{up}-to:{down}~";
return test.GenerateTestFile(testFileSize, label);
return fileManager.GenerateTestFile(testFileSize, label);
}
}
}

197
DistTestCore/Metrics/dashboard.json
View File

@ -512,6 +512,203 @@
"title": "Bytes used",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 2
},
"id": 24,
"panels": [],
"title": "Network Status",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 3
},
"id": 25,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb"
},
"editorMode": "builder",
"expr": "libp2p_peers",
"instant": false,
"range": true,
"refId": "A"
}
],
"title": "Peers",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 3
},
"id": 26,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb"
},
"editorMode": "builder",
"expr": "routing_table_nodes",
"instant": false,
"range": true,
"refId": "A"
}
],
"title": "Routing table nodes",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {

18
KubernetesWorkflow/CrashWatcher.cs
View File

@ -43,6 +43,12 @@ namespace KubernetesWorkflow
if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
}
public bool HasContainerCrashed()
{
using var client = new Kubernetes(config);
return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
}
private void Worker()
{
try
@ -57,7 +63,7 @@ namespace KubernetesWorkflow
private void MonitorContainer(CancellationToken token)
{
var client = new Kubernetes(config);
using var client = new Kubernetes(config);
while (!token.IsCancellationRequested)
{
token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
@ -65,9 +71,7 @@ namespace KubernetesWorkflow
var pod = container.Pod;
var recipe = container.Recipe;
var podName = pod.PodInfo.Name;
var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
if (HasContainerBeenRestarted(client, podName))
{
DownloadCrashedContainerLogs(client, podName, recipe);
return;
@ -75,6 +79,12 @@ namespace KubernetesWorkflow
}
}
private bool HasContainerBeenRestarted(Kubernetes client, string podName)
{
var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
}
private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
{
log.Log("Pod crash detected for " + container.Name);

5
LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore.Helpers;
using DistTestCore;
using DistTestCore;
using NUnit.Framework;
namespace TestsLong.DownloadConnectivityTests
@ -16,7 +15,7 @@ namespace TestsLong.DownloadConnectivityTests
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}
}
}

11
Tests/BasicTests/TwoClientTests.cs
View File

@ -8,7 +8,7 @@ namespace Tests.BasicTests
public class TwoClientTests : DistTest
{
[Test]
public void TwoClientsOnePodTest()
public void TwoClientTest()
{
var group = SetupCodexNodes(2);
@ -18,15 +18,6 @@ namespace Tests.BasicTests
PerformTwoClientTest(primary, secondary);
}
[Test]
public void TwoClientsTwoPodsTest()
{
var primary = SetupCodexNode();
var secondary = SetupCodexNode();
PerformTwoClientTest(primary, secondary);
}
[Test]
public void TwoClientsTwoLocationsTest()
{

25
Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs
View File

@ -6,15 +6,36 @@ namespace Tests.DownloadConnectivityTests
[TestFixture]
public class FullyConnectedDownloadTests : AutoBootstrapDistTest
{
[Test]
public void MetricsDoesNotInterfereWithPeerDownload()
{
SetupCodexNodes(2, s => s.EnableMetrics());
AssertAllNodesConnected();
}
[Test]
public void MarketplaceDoesNotInterfereWithPeerDownload()
{
SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
AssertAllNodesConnected();
}
[Test]
[Combinatorial]
public void FullyConnectedDownloadTest(
[Values(1, 3, 5)] int numberOfNodes,
[Values(1, 10)] int sizeMBs)
{
for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
SetupCodexNodes(numberOfNodes);
PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
AssertAllNodesConnected(sizeMBs);
}
private void AssertAllNodesConnected(int sizeMBs = 10)
{
CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
}
}
}

3
Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
namespace Tests.PeerDiscoveryTests
@ -47,7 +46,7 @@ namespace Tests.PeerDiscoveryTests
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
}
}
}

26
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
namespace Tests.PeerDiscoveryTests
@ -17,23 +16,36 @@ namespace Tests.PeerDiscoveryTests
Assert.That(result.IsPeerFound, Is.False);
}
[Test]
public void MetricsDoesNotInterfereWithPeerDiscovery()
{
SetupCodexNodes(2, s => s.EnableMetrics());
AssertAllNodesConnected();
}
[Test]
public void MarketplaceDoesNotInterfereWithPeerDiscovery()
{
SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens()));
AssertAllNodesConnected();
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
[TestCase(20)]
public void VariableNodesInPods(int number)
public void VariableNodes(int number)
{
for (var i = 0; i < number; i++)
{
SetupCodexNode();
}
SetupCodexNodes(number);
AssertAllNodesConnected();
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
}
}
}