status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/location'

This commit is contained in:
benbierens 2023-03-27 07:58:10 +02:00
parent 6fc3ea3fca ed2878f7df
commit 8c16ae3385
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
16 changed files with 329 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
CodexDistTestCore/CodexNodeGroup.cs
View File

@ -1,4 +1,5 @@
using k8s.Models;
using CodexDistTestCore.Config;
using k8s.Models;
using System.Collections;
namespace CodexDistTestCore
@ -11,10 +12,12 @@ namespace CodexDistTestCore
public class CodexNodeGroup : ICodexNodeGroup
{
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
{
this.log = log;
OrderNumber = orderNumber;
Origin = origin;
this.k8SManager = k8SManager;
@ -63,7 +66,7 @@ namespace CodexDistTestCore
return new V1ObjectMeta
{
Name = "codex-test-entrypoint-" + OrderNumber,
NamespaceProperty = K8sOperations.K8sNamespace
NamespaceProperty = K8sCluster.K8sNamespace
};
}
@ -72,10 +75,17 @@ namespace CodexDistTestCore
return new V1ObjectMeta
{
Name = "codex-test-node-" + OrderNumber,
NamespaceProperty = K8sOperations.K8sNamespace
NamespaceProperty = K8sCluster.K8sNamespace
};
}
public CodexNodeLog DownloadLog(IOnlineCodexNode node)
{
var logDownloader = new PodLogDownloader(log, k8SManager);
var n = (OnlineCodexNode)node;
return logDownloader.DownloadLog(n);
}
public Dictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + OrderNumber } };

29
CodexDistTestCore/CodexNodeLog.cs Normal file
View File

@ -0,0 +1,29 @@
using NUnit.Framework;
namespace CodexDistTestCore
{
public class CodexNodeLog
{
private readonly LogFile logFile;
public CodexNodeLog(LogFile logFile)
{
this.logFile = logFile;
}
public void AssertLogContains(string expectedString)
{
using var file = File.OpenRead(logFile.FullFilename);
using var streamReader = new StreamReader(file);
var line = streamReader.ReadLine();
while (line != null)
{
if (line.Contains(expectedString)) return;
line = streamReader.ReadLine();
}
Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
}
}
}

2
CodexDistTestCore/CodexDockerImage.cs → CodexDistTestCore/Config/CodexDockerImage.cs
View File

@ -1,6 +1,6 @@
using k8s.Models;
namespace CodexDistTestCore
namespace CodexDistTestCore.Config
{
public class CodexDockerImage
{

7
CodexDistTestCore/Config/FileManagerConfig.cs Normal file
View File

@ -0,0 +1,7 @@
namespace CodexDistTestCore.Config
{
public class FileManagerConfig
{
public const string Folder = "TestDataFiles";
}
}

39
CodexDistTestCore/Config/K8sCluster.cs Normal file
View File

@ -0,0 +1,39 @@
using k8s;
namespace CodexDistTestCore.Config
{
public class K8sCluster
{
public const string K8sNamespace = "codex-test-namespace";
private const string KubeConfigFile = "C:\\kube\\config";
private readonly Dictionary<Location, string> K8sNodeLocationMap = new Dictionary<Location, string>
{
{ Location.BensLaptop, "worker01" },
{ Location.BensOldGamingMachine, "worker02" },
};
private KubernetesClientConfiguration? config;
public KubernetesClientConfiguration GetK8sClientConfig()
{
if (config != null) return config;
config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
return config;
}
public string GetIp()
{
var c = GetK8sClientConfig();
var host = c.Host.Replace("https://", "");
return host.Substring(0, host.IndexOf(':'));
}
public string GetNodeLabelForLocation(Location location)
{
if (location == Location.Unspecified) return string.Empty;
return K8sNodeLocationMap[location];
}
}
}

7
CodexDistTestCore/Config/LogConfig.cs Normal file
View File

@ -0,0 +1,7 @@
namespace CodexDistTestCore.Config
{
public class LogConfig
{
public const string LogRoot = "D:/CodexTestLogs";
}
}

42
CodexDistTestCore/DistTest.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
@ -41,7 +42,10 @@ namespace CodexDistTestCore
}
else
{
var dockerImage = new CodexDockerImage();
log = new TestLog();
log.Log($"Using docker image '{dockerImage.GetImageTag()}'");
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
}
@ -52,7 +56,8 @@ namespace CodexDistTestCore
{
try
{
log.EndTest(k8sManager);
log.EndTest();
IncludeLogsOnTestFailure();
k8sManager.DeleteAllResources();
fileManager.DeleteAllTestFiles();
}
@ -72,6 +77,39 @@ namespace CodexDistTestCore
{
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
private void IncludeLogsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
if (IsDownloadingLogsEnabled())
{
log.Log("Downloading all CodexNode logs because of test failure...");
k8sManager.ForEachOnlineGroup(DownloadLogs);
}
else
{
log.Log("Skipping download of all CodexNode logs due to [DontDownloadLogsOnFailure] attribute.");
}
}
}
private void DownloadLogs(CodexNodeGroup group)
{
foreach (var node in group)
{
var downloader = new PodLogDownloader(log, k8sManager);
var n = (OnlineCodexNode)node;
downloader.DownloadLog(n);
}
}
private bool IsDownloadingLogsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
}
}
public static class GlobalTestFailure

8
CodexDistTestCore/FileManager.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
@ -12,20 +13,19 @@ namespace CodexDistTestCore
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024;
private const string Folder = "TestDataFiles";
private readonly Random random = new Random();
private readonly List<TestFile> activeFiles = new List<TestFile>();
private readonly TestLog log;
public FileManager(TestLog log)
{
if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder);
if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder);
this.log = log;
}
public TestFile CreateEmptyTestFile()
{
var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
activeFiles.Add(result);
return result;

18
CodexDistTestCore/K8sManager.cs
View File

@ -4,12 +4,13 @@
{
ICodexNodeGroup BringOnline(OfflineCodexNodes node);
IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
}
public class K8sManager : IK8sManager
{
private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
private readonly List<CodexNodeGroup> onlineCodexNodeGroups = new List<CodexNodeGroup>();
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
@ -47,17 +48,22 @@
K8s(k => k.DeleteAllResources());
}
public void FetchAllPodsLogs(IPodLogsHandler logHandler)
public void ForEachOnlineGroup(Action<CodexNodeGroup> action)
{
K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler));
foreach (var group in onlineCodexNodeGroups) action(group);
}
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
K8s(k => k.FetchPodLog(node, logHandler));
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline.NumberOfNodes);
var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodes.Add(result);
var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
onlineCodexNodeGroups.Add(result);
return result;
}
@ -72,7 +78,7 @@
private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
{
var n = (CodexNodeGroup)node;
onlineCodexNodes.Remove(n);
onlineCodexNodeGroups.Remove(n);
return n;
}

47
CodexDistTestCore/K8sOperations.cs
View File

@ -1,4 +1,5 @@
using k8s;
using CodexDistTestCore.Config;
using k8s;
using k8s.Models;
using NUnit.Framework;
@ -6,9 +7,8 @@ namespace CodexDistTestCore
{
public class K8sOperations
{
public const string K8sNamespace = "codex-test-namespace";
private readonly CodexDockerImage dockerImage = new CodexDockerImage();
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Kubernetes client;
private readonly KnownK8sPods knownPods;
@ -16,9 +16,7 @@ namespace CodexDistTestCore
{
this.knownPods = knownPods;
// todo: If the default KubeConfig file does not suffice, change it here:
var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
client = new Kubernetes(config);
client = new Kubernetes(k8sCluster.GetK8sClientConfig());
}
public void Close()
@ -53,25 +51,10 @@ namespace CodexDistTestCore
WaitUntilNamespaceDeleted();
}
public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
{
var logNumberSource = new NumberSource(0);
foreach (var online in onlines)
{
foreach (var node in online)
{
WritePodLogs(online, node, logHandler, logNumberSource);
}
}
}
private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource)
{
var n = (OnlineCodexNode)node;
var nodeDescription = $"{online.Describe()} contains {n.GetName()}";
var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name);
logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream);
var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
logHandler.Log(stream);
}
private void FetchPodInfo(CodexNodeGroup online)
@ -208,6 +191,7 @@ namespace CodexDistTestCore
},
Spec = new V1PodSpec
{
NodeSelector = CreateNodeSelector(offline),
Containers = CreateDeploymentContainers(online, offline)
}
}
@ -217,6 +201,16 @@ namespace CodexDistTestCore
online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
}
private IDictionary<string, string> CreateNodeSelector(OfflineCodexNodes offline)
{
if (offline.Location == Location.Unspecified) return new Dictionary<string, string>();
return new Dictionary<string, string>
{
{ "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) }
};
}
private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
{
var result = new List<V1Container>();
@ -276,6 +270,11 @@ namespace CodexDistTestCore
}
}
private string K8sNamespace
{
get { return K8sCluster.K8sNamespace; }
}
#endregion
private bool IsTestNamespaceOnline()

16
CodexDistTestCore/OfflineCodexNodes.cs
View File

@ -2,6 +2,7 @@
{
public interface IOfflineCodexNodes
{
IOfflineCodexNodes At(Location location);
IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
@ -17,11 +18,19 @@
Error
}
public enum Location
{
Unspecified,
BensLaptop,
BensOldGamingMachine,
}
public class OfflineCodexNodes : IOfflineCodexNodes
{
private readonly IK8sManager k8SManager;
public int NumberOfNodes { get; }
public Location Location { get; private set; }
public CodexLogLevel? LogLevel { get; private set; }
public IOnlineCodexNode? BootstrapNode { get; private set; }
public ByteSize? StorageQuota { get; private set; }
@ -30,6 +39,7 @@
{
this.k8SManager = k8SManager;
NumberOfNodes = numberOfNodes;
Location = Location.Unspecified;
}
public ICodexNodeGroup BringOnline()
@ -37,6 +47,12 @@
return k8SManager.BringOnline(this);
}
public IOfflineCodexNodes At(Location location)
{
Location = location;
return this;
}
public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
{
BootstrapNode = node;

17
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -1,4 +1,5 @@
using NUnit.Framework;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
@ -8,6 +9,7 @@ namespace CodexDistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
CodexNodeLog DownloadLog();
}
public class OnlineCodexNode : IOnlineCodexNode
@ -15,6 +17,7 @@ namespace CodexDistTestCore
private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
private const string UploadFailedMessage = "Unable to store block";
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly TestLog log;
private readonly IFileManager fileManager;
@ -77,6 +80,16 @@ namespace CodexDistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public CodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
}
public string Describe()
{
return $"{Group.Describe()} contains {GetName()}";
}
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
{
var multiAddress = peerInfo.addrs.First();
@ -101,7 +114,7 @@ namespace CodexDistTestCore
private Http Http()
{
return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1");
}
private void Log(string msg)

73
CodexDistTestCore/PodLogDownloader.cs Normal file
View File

@ -0,0 +1,73 @@
using NUnit.Framework;
namespace CodexDistTestCore
{
public interface IPodLogHandler
{
void Log(Stream log);
}
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
{
public DontDownloadLogsOnFailureAttribute()
: base(Timing.UseLongTimeoutsKey)
{
}
}
public class PodLogDownloader
{
public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure";
private readonly TestLog log;
private readonly IK8sManager k8SManager;
public PodLogDownloader(TestLog log, IK8sManager k8sManager)
{
this.log = log;
k8SManager = k8sManager;
}
public CodexNodeLog DownloadLog(OnlineCodexNode node)
{
var description = node.Describe();
var subFile = log.CreateSubfile();
log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}");
var handler = new PodLogDownloadHandler(description, subFile);
k8SManager.FetchPodLog(node, handler);
return handler.CreateCodexNodeLog();
}
}
public class PodLogDownloadHandler : IPodLogHandler
{
private readonly string description;
private readonly LogFile log;
public PodLogDownloadHandler(string description, LogFile log)
{
this.description = description;
this.log = log;
}
public CodexNodeLog CreateCodexNodeLog()
{
return new CodexNodeLog(log);
}
public void Log(Stream stream)
{
log.Write($"{description} -->> {log.FilenameWithoutPath}");
log.WriteRaw(description);
var reader = new StreamReader(stream);
var line = reader.ReadLine();
while (line != null)
{
log.WriteRaw(line);
line = reader.ReadLine();
}
}
}
}

7
CodexDistTestCore/PodLogsHandler.cs
View File

@ -1,7 +0,0 @@
namespace CodexDistTestCore
{
public interface IPodLogsHandler
{
void Log(int id, string podDescription, Stream log);
}
}

77
CodexDistTestCore/TestLog.cs
View File

@ -1,16 +1,20 @@
using NUnit.Framework;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace CodexDistTestCore
{
public class TestLog
{
public const string LogRoot = "D:/CodexTestLogs";
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly LogFile file;
private readonly DateTime now;
public TestLog()
{
now = DateTime.UtcNow;
var name = GetTestName();
file = new LogFile(name);
file = new LogFile(now, name);
Log($"Begin: {name}");
}
@ -25,26 +29,23 @@ namespace CodexDistTestCore
Log($"[ERROR] {message}");
}
public void EndTest(K8sManager k8sManager)
public void EndTest()
{
var result = TestContext.CurrentContext.Result;
Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
Log($"{result.StackTrace}");
var logWriter = new PodLogWriter(file);
logWriter.IncludeFullPodLogging(k8sManager);
}
}
public LogFile CreateSubfile()
{
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}");
}
private static string GetTestName()
{
var test = TestContext.CurrentContext.Test;
@ -60,70 +61,36 @@ namespace CodexDistTestCore
}
}
public class PodLogWriter : IPodLogsHandler
{
private readonly LogFile file;
public PodLogWriter(LogFile file)
{
this.file = file;
}
public void IncludeFullPodLogging(K8sManager k8sManager)
{
file.Write("Full pod logging:");
k8sManager.FetchAllPodsLogs(this);
}
public void Log(int id, string podDescription, Stream log)
{
var logFile = id.ToString().PadLeft(6, '0');
file.Write($"{podDescription} -->> {logFile}");
LogRaw(podDescription, logFile);
var reader = new StreamReader(log);
var line = reader.ReadLine();
while (line != null)
{
LogRaw(line, logFile);
line = reader.ReadLine();
}
}
private void LogRaw(string message, string filename)
{
file!.WriteRaw(message, filename);
}
}
public class LogFile
{
private readonly string filepath;
private readonly string filename;
public LogFile(string name)
public LogFile(DateTime now, string name)
{
var now = DateTime.UtcNow;
filepath = Path.Join(
TestLog.LogRoot,
LogConfig.LogRoot,
$"{now.Year}-{Pad(now.Month)}",
Pad(now.Day));
Directory.CreateDirectory(filepath);
filename = Path.Combine(filepath, $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}");
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
}
public string FullFilename { get; }
public string FilenameWithoutPath { get; }
public void Write(string message)
{
WriteRaw($"{GetTimestamp()} {message}");
}
public void WriteRaw(string message, string subfile = "")
public void WriteRaw(string message)
{
try
{
File.AppendAllLines(filename + subfile + ".log", new[] { message });
File.AppendAllLines(FullFilename, new[] { message });
}
catch (Exception ex)
{

39
Tests/BasicTests/SimpleTests.cs
View File

@ -1,4 +1,5 @@
using CodexDistTestCore;
using CodexDistTestCore.Config;
using NUnit.Framework;
namespace Tests.BasicTests
@ -19,13 +20,20 @@ namespace Tests.BasicTests
Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision()));
}
[Test, DontDownloadLogsOnFailure]
public void CanAccessLogs()
{
var node = SetupCodexNodes(1).BringOnline()[0];
var log = node.DownloadLog();
log.AssertLogContains("Started codex node");
}
[Test]
public void OneClientTest()
{
var primary = SetupCodexNodes(1)
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(2.MB())
.BringOnline()[0];
var primary = SetupCodexNodes(1).BringOnline()[0];
var testFile = GenerateTestFile(1.MB());
@ -37,12 +45,9 @@ namespace Tests.BasicTests
}
[Test]
public void TwoClientOnePodTest()
public void TwoClientsOnePodTest()
{
var group = SetupCodexNodes(2)
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(2.MB())
.BringOnline();
var group = SetupCodexNodes(2).BringOnline();
var primary = group[0];
var secondary = group[1];
@ -51,14 +56,24 @@ namespace Tests.BasicTests
}
[Test]
public void TwoClientTwoPodTest()
public void TwoClientsTwoPodsTest()
{
var primary = SetupCodexNodes(1).BringOnline()[0];
var secondary = SetupCodexNodes(1).BringOnline()[0];
PerformTwoClientTest(primary, secondary);
}
[Test]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)
.WithStorageQuota(2.MB())
.At(Location.BensLaptop)
.BringOnline()[0];
var secondary = SetupCodexNodes(1)
.WithStorageQuota(2.MB())
.At(Location.BensOldGamingMachine)
.BringOnline()[0];
PerformTwoClientTest(primary, secondary);