codex-storage/cs-codex-dist-tests
1
0
Fork 0
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git synced 2025-02-09 10:45:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adds peer connectivity check after deployment

Browse Source
This commit is contained in:
benbierens 2023-08-24 11:32:32 +02:00
parent 7eda26d177
commit eec70a9b8c
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
8 changed files with 103 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -21,7 +21,7 @@ namespace CodexNetDeployer
validatorsLeft = numberOfValidators;
}
public RunningContainer? Start(int i)
public CodexNodeStartResult? Start(int i)
{
Console.Write($" - {i} = ");
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
@ -60,7 +60,7 @@ namespace CodexNetDeployer
if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
validatorsLeft--;
return container;
return new CodexNodeStartResult(container, codexAccess);
}
}
}
@ -102,4 +102,16 @@ namespace CodexNetDeployer
return codexStart;
}
}
public class CodexNodeStartResult
{
public CodexNodeStartResult(RunningContainer container, CodexAccess access)
{
Container = container;
Access = access;
}
public RunningContainer Container { get; }
public CodexAccess Access { get; }
}
}

3
CodexNetDeployer/Configuration.cs
View File

@ -57,6 +57,9 @@ namespace CodexNetDeployer
[Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
"set this option to override the label value.")]
public string TestsTypePodLabel { get; set; } = "continuous-tests";
[Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
public bool CheckPeerConnection { get; set; } = false;
public List<string> Validate()
{

29
CodexNetDeployer/Deployer.cs
View File

@ -8,14 +8,14 @@ namespace CodexNetDeployer
public class Deployer
{
private readonly Configuration config;
private readonly NullLog log;
private readonly DefaultTimeSet timeset;
private readonly PeerConnectivityChecker peerConnectivityChecker;
public Deployer(Configuration config)
{
this.config = config;
log = new NullLog();
timeset = new DefaultTimeSet();
peerConnectivityChecker = new PeerConnectivityChecker();
}
public CodexDeployment Deploy()
@ -45,16 +45,18 @@ namespace CodexNetDeployer
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexContainers = new List<RunningContainer>();
var startResults = new List<CodexNodeStartResult>();
for (var i = 0; i < config.NumberOfCodexNodes; i++)
{
var container = codexStarter.Start(i);
if (container != null) codexContainers.Add(container);
var result = codexStarter.Start(i);
if (result != null) startResults.Add(result);
}
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers);
var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
CheckPeerConnectivity(startResults);
return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
}
private TestLifecycle CreateTestLifecycle()
@ -71,10 +73,10 @@ namespace CodexNetDeployer
k8sNamespacePrefix: config.KubeNamespace
);
return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
}
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
{
if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
@ -95,6 +97,15 @@ namespace CodexNetDeployer
return kubeConfigFile;
}
private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
{
if (!config.CheckPeerConnection) return;
Log("Starting peer-connectivity check for deployed nodes...");
peerConnectivityChecker.CheckConnectivity(codexContainers);
Log("Check passed.");
}
private DeploymentMetadata CreateMetadata()
{
return new DeploymentMetadata(

34
CodexNetDeployer/PeerConnectivityChecker.cs Normal file
View File

@ -0,0 +1,34 @@
using DistTestCore.Helpers;
using Logging;
namespace CodexNetDeployer
{
public class PeerConnectivityChecker
{
public void CheckConnectivity(List<CodexNodeStartResult> startResults)
{
var checker = new PeerConnectionTestHelpers(new ConsoleLogger());
var access = startResults.Select(r => r.Access);
checker.AssertFullyConnected(access);
}
}
public class ConsoleLogger : BaseLog
{
public ConsoleLogger() : base(false)
{
}
protected override string GetFullName()
{
return "CONSOLE";
}
public override void Log(string message)
{
Console.WriteLine(message);
}
}
}

5
DistTestCore/Codex/CodexAccess.cs
View File

@ -88,6 +88,11 @@ namespace DistTestCore.Codex
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
public string GetName()
{
return Container.Name;
}
private Http Http()
{
CheckContainerCrashed();

10
DistTestCore/Helpers/FullConnectivityHelper.cs
View File

@ -24,12 +24,12 @@ namespace DistTestCore.Helpers
this.implementation = implementation;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
private void AssertFullyConnected(IOnlineCodexNode[] nodes)
private void AssertFullyConnected(CodexAccess[] nodes)
{
Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
@ -85,7 +85,7 @@ namespace DistTestCore.Helpers
Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
}
private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
private Entry[] CreateEntries(CodexAccess[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
@ -125,13 +125,13 @@ namespace DistTestCore.Helpers
public class Entry
{
public Entry(IOnlineCodexNode node)
public Entry(CodexAccess node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexAccess Node { get; }
public CodexDebugResponse Response { get; }
public override string ToString()

10
DistTestCore/Helpers/PeerConnectionTestHelpers.cs
View File

@ -14,6 +14,11 @@ namespace DistTestCore.Helpers
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
}
public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
{
helper.AssertFullyConnected(nodes);
}
@ -58,9 +63,8 @@ namespace DistTestCore.Helpers
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
var ip = peer.Node.Container.Pod.PodInfo.Ip;
var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}

19
DistTestCore/Helpers/PeerDownloadTestHelpers.cs
View File

@ -1,4 +1,5 @@
using Logging;
using DistTestCore.Codex;
using Logging;
using static DistTestCore.Helpers.FullConnectivityHelper;
namespace DistTestCore.Helpers
@ -17,6 +18,11 @@ namespace DistTestCore.Helpers
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
}
public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
{
this.testFileSize = testFileSize;
helper.AssertFullyConnected(nodes);
@ -37,11 +43,16 @@ namespace DistTestCore.Helpers
fileManager.PushFileSet();
var expectedFile = GenerateTestFile(from.Node, to.Node);
var contentId = from.Node.UploadFile(expectedFile);
using var uploadStream = File.OpenRead(expectedFile.Filename);
var contentId = from.Node.UploadFile(uploadStream);
try
{
var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
var downloadedFile = fileManager.CreateEmptyTestFile(expectedFile.Label + "_downloaded");
using var downloadStream = File.OpenWrite(downloadedFile.Filename);
using var stream = to.Node.DownloadFile(contentId);
stream.CopyTo(downloadStream);
expectedFile.AssertIsEqual(downloadedFile);
return PeerConnectionState.Connection;
}
@ -59,7 +70,7 @@ namespace DistTestCore.Helpers
// Should an exception occur during upload, then this try is inconclusive and we try again next loop.
}
private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
{
var up = uploader.GetName().Replace("<", "").Replace(">", "");
var down = downloader.GetName().Replace("<", "").Replace(">", "");