status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/continuous-peer-test'

This commit is contained in:
benbierens 2023-09-04 09:15:30 +02:00
parent a4b6b561d7 c673aff0fe
commit a528c0111d
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
15 changed files with 131 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -28,6 +28,7 @@ namespace CodexNetDeployer
var workflowStartup = new StartupConfig();
workflowStartup.Add(gethResult);
workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
workflowStartup.NameOverride = GetCodexContainerName(i);
var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
@ -75,6 +76,12 @@ namespace CodexNetDeployer
return null;
}
private string GetCodexContainerName(int i)
{
if (i == 0) return "BOOTSTRAP";
return "CODEX" + i;
}
private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
{
var codexStart = new CodexStartupConfig(config.CodexLogLevel);

4
ContinuousTests/Configuration.cs
View File

@ -22,8 +22,8 @@ namespace ContinuousTests
[Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
public string KubeConfigFile { get; set; } = "null";
[Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
public bool StopOnFailure { get; set; } = false;
[Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
public int StopOnFailure { get; set; } = 0;
[Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
public bool DownloadContainerLogs { get; set; } = false;

2
ContinuousTests/ContinuousTestRunner.cs
View File

@ -30,7 +30,7 @@ namespace ContinuousTests
ClearAllCustomNamespaces(allTests, overviewLog);
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
foreach (var testLoop in testLoops)
{

75
ContinuousTests/SingleTestRun.cs
View File

@ -23,8 +23,9 @@ namespace ContinuousTests
private readonly FixtureLog fixtureLog;
private readonly string testName;
private readonly string dataFolder;
private static int failureCount = 0;
public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
{
this.taskFactory = taskFactory;
this.config = config;
@ -33,8 +34,9 @@ namespace ContinuousTests
this.cancelToken = cancelToken;
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
ApplyLogReplacements(fixtureLog, startupChecker);
nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
nodes = CreateRandomNodes();
dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
}
@ -71,16 +73,26 @@ namespace ContinuousTests
fixtureLog.Error("Test run failed with exception: " + ex);
fixtureLog.MarkAsFailed();
if (config.StopOnFailure)
failureCount++;
if (config.StopOnFailure > 0)
{
OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
DownloadClusterLogs();
OverviewLog("Log download finished. Cancelling test runner...");
Cancellation.Cts.Cancel();
OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
if (failureCount >= config.StopOnFailure)
{
OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
DownloadClusterLogs();
OverviewLog("Log download finished. Cancelling test runner...");
Cancellation.Cts.Cancel();
}
}
}
}
private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
{
foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
}
private void RunTestMoments()
{
var earliestMoment = handle.GetEarliestMoment();
@ -112,7 +124,7 @@ namespace ContinuousTests
{
ThrowFailTest();
}
OverviewLog(" > Test passed. " + FuturesInfo());
OverviewLog(" > Test passed.");
return;
}
}
@ -120,25 +132,19 @@ namespace ContinuousTests
private void ThrowFailTest()
{
var ex = UnpackException(exceptions.First());
Log(ex.ToString());
OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
throw ex;
}
private string FuturesInfo()
{
var containers = config.CodexDeployment.CodexContainers;
var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
var f = nodes.Select(n => n.GetDebugFutures().ToString());
var msg = $"(Futures: [{string.Join(", ", f)}])";
return msg;
var exs = UnpackExceptions(exceptions);
var exceptionsMessage = GetCombinedExceptionsMessage(exs);
Log(exceptionsMessage);
OverviewLog($" > Test failed: " + exceptionsMessage);
throw new Exception(exceptionsMessage);
}
private void DownloadClusterLogs()
{
var k8sFactory = new K8sFactory();
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
var log = new NullLog();
log.FullFilename = Path.Combine(config.LogPath, "NODE");
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
foreach (var container in config.CodexDeployment.CodexContainers)
{
@ -146,6 +152,16 @@ namespace ContinuousTests
}
}
private string GetCombinedExceptionsMessage(Exception[] exceptions)
{
return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
}
private Exception[] UnpackExceptions(List<Exception> exceptions)
{
return exceptions.Select(UnpackException).ToArray();
}
private Exception UnpackException(Exception exception)
{
if (exception is AggregateException a)
@ -196,19 +212,26 @@ namespace ContinuousTests
private void OverviewLog(string msg)
{
Log(msg);
var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
var containerNames = GetContainerNames();
overviewLog.Log($"{containerNames} {testName}: {msg}");
}
private CodexAccess[] CreateRandomNodes(int number)
private string GetContainerNames()
{
var containers = SelectRandomContainers(number);
if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
}
private CodexAccess[] CreateRandomNodes()
{
var containers = SelectRandomContainers();
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
}
private RunningContainer[] SelectRandomContainers(int number)
private RunningContainer[] SelectRandomContainers()
{
var number = handle.Test.RequiredNumberOfNodes;
if (number == -1) return config.CodexDeployment.CodexContainers;
var containers = config.CodexDeployment.CodexContainers.ToList();

4
ContinuousTests/StartupChecker.cs
View File

@ -15,6 +15,7 @@ namespace ContinuousTests
{
this.config = config;
this.cancelToken = cancelToken;
LogReplacements = new List<BaseLogStringReplacement>();
}
public void Check()
@ -28,6 +29,8 @@ namespace ContinuousTests
log.Log("All OK.");
}
public List<BaseLogStringReplacement> LogReplacements { get; }
private void PreflightCheck(Configuration config)
{
var tests = testFactory.CreateTests();
@ -90,6 +93,7 @@ namespace ContinuousTests
if (info == null || string.IsNullOrEmpty(info.id)) return false;
log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName()));
}
catch
{

6
ContinuousTests/TestLoop.cs
View File

@ -9,16 +9,18 @@ namespace ContinuousTests
private readonly BaseLog overviewLog;
private readonly Type testType;
private readonly TimeSpan runsEvery;
private readonly StartupChecker startupChecker;
private readonly CancellationToken cancelToken;
private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
{
this.taskFactory = taskFactory;
this.config = config;
this.overviewLog = overviewLog;
this.testType = testType;
this.runsEvery = runsEvery;
this.startupChecker = startupChecker;
this.cancelToken = cancelToken;
Name = testType.Name;
}
@ -58,7 +60,7 @@ namespace ContinuousTests
{
var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test);
var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
runFinishedHandle.Reset();
run.Run(runFinishedHandle);

12
ContinuousTests/Tests/HoldMyBeerTest.cs
View File

@ -15,17 +15,7 @@ namespace ContinuousTests.Tests
[TestMoment(t: Zero)]
public void UploadTestFile()
{
var metadata = Configuration.CodexDeployment.Metadata;
var maxQuotaUseMb = metadata.StorageQuotaMB / 2;
var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30;
var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds);
var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl);
// This filesize should keep the quota below 50% of the node's max.
var filesize = filesizePerUploadMb.MB();
double codexDefaultBlockSize = 31 * 64 * 33;
var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
Assert.That(numberOfBlocks, Is.EqualTo(1282));
var filesize = 80.MB();
file = FileManager.GenerateTestFile(filesize);

22
ContinuousTests/Tests/PeersTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using DistTestCore.Helpers;
using NUnit.Framework;
namespace ContinuousTests.Tests
@ -10,10 +11,24 @@ namespace ContinuousTests.Tests
public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
[TestMoment(t: 0)]
public void CheckConnectivity()
{
var checker = new PeerConnectionTestHelpers(Log);
checker.AssertFullyConnected(Nodes);
}
[TestMoment(t: 10)]
public void CheckRoutingTables()
{
var allIds = Nodes.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
var allInfos = Nodes.Select(n =>
{
var info = n.GetDebugInfo();
Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}");
Log.AddStringReplace(info.table.localNode.nodeId, n.GetName());
return info;
}).ToArray();
var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray();
var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
if (errors.Any())
@ -30,7 +45,10 @@ namespace ContinuousTests.Tests
if (!expected.All(ex => known.Contains(ex)))
{
return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
var nl = Environment.NewLine;
return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" +
$"Not all of{nl}'{string.Join(",", expected)}'{nl}" +
$"were present in routing table:{nl}'{string.Join(",", known)}'";
}
return string.Empty;

2
ContinuousTests/run.sh
View File

@ -2,5 +2,5 @@ dotnet run \
--kube-config=/opt/kubeconfig.yaml \
--codex-deployment=codex-deployment.json \
--keep=1 \
--stop=1 \
--stop=10 \
--dl-logs=1

8
DistTestCore/Codex/CodexAccess.cs
View File

@ -47,12 +47,6 @@ namespace DistTestCore.Codex
return result;
}
public int GetDebugFutures()
{
// Some Codex images support debug/futures to count the number of open futures.
return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
}
public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
{
return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
@ -75,7 +69,7 @@ namespace DistTestCore.Codex
public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
{
return Http().HttpPostJson<CodexSalesRequestStorageRequest, string>($"storage/request/{contentId}", request);
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)

5
DistTestCore/Codex/CodexApiTypes.cs
View File

@ -1,7 +1,4 @@
using KubernetesWorkflow;
using Logging;
using Newtonsoft.Json;
using Utils;
using Newtonsoft.Json;
namespace DistTestCore.Codex
{

38
DistTestCore/Http.cs
View File

@ -55,25 +55,20 @@ namespace DistTestCore
public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
{
var response = HttpPostJson(route, body);
var response = PostJson(route, body);
var json = Time.Wait(response.Content.ReadAsStringAsync());
if(!response.IsSuccessStatusCode) {
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException(json);
}
Log(GetUrl() + route, json);
return TryJsonDeserialize<TResponse>(json);
}
public HttpResponseMessage HttpPostJson<TRequest>(string route, TRequest body)
public string HttpPostJson<TRequest>(string route, TRequest body)
{
return Retry(() =>
{
using var client = GetClient();
var url = GetUrl() + route;
using var content = JsonContent.Create(body);
Log(url, JsonConvert.SerializeObject(body));
return Time.Wait(client.PostAsync(url, content));
}, $"HTTP-POST-JSON: {route}");
var response = PostJson<TRequest>(route, body);
return Time.Wait(response.Content.ReadAsStringAsync());
}
public string HttpPostString(string route, string body)
@ -122,7 +117,8 @@ namespace DistTestCore
public T TryJsonDeserialize<T>(string json)
{
var errors = new List<string>();
var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings(){
var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
{
Error = delegate(object? sender, Serialization.ErrorEventArgs args)
{
if (args.CurrentObject == args.ErrorContext.OriginalObject)
@ -136,15 +132,29 @@ namespace DistTestCore
}
}
});
if (errors.Count() > 0) {
if (errors.Count() > 0)
{
throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
}
else if (deserialized == null) {
else if (deserialized == null)
{
throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
}
return deserialized;
}
private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
{
return Retry(() =>
{
using var client = GetClient();
var url = GetUrl() + route;
using var content = JsonContent.Create(body);
Log(url, JsonConvert.SerializeObject(body));
return Time.Wait(client.PostAsync(url, content));
}, $"HTTP-POST-JSON: {route}");
}
private string GetUrl()
{
return $"{address.Host}:{address.Port}{baseUrl}";

13
Logging/BaseLog.cs
View File

@ -60,6 +60,7 @@ namespace Logging
public virtual void AddStringReplace(string from, string to)
{
if (string.IsNullOrWhiteSpace(from)) return;
if (replacements.Any(r => r.From == from)) return;
replacements.Add(new BaseLogStringReplacement(from, to));
}
@ -98,20 +99,20 @@ namespace Logging
public class BaseLogStringReplacement
{
private readonly string from;
private readonly string to;
public BaseLogStringReplacement(string from, string to)
{
this.from = from;
this.to = to;
From = from;
To = to;
if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
}
public string From { get; }
public string To { get; }
public string Apply(string msg)
{
return msg.Replace(from, to);
return msg.Replace(From, To);
}
}
}

4
Logging/NullLog.cs
View File

@ -6,9 +6,11 @@
{
}
public string FullFilename { get; set; } = "NULL";
protected override string GetFullName()
{
return "NULL";
return FullFilename;
}
public override void Log(string message)

11
Tests/BasicTests/ContinuousSubstitute.cs
View File

@ -4,6 +4,7 @@ using Utils;
namespace Tests.BasicTests
{
[Ignore("Used for debugging continuous tests")]
[TestFixture]
public class ContinuousSubstitute : AutoBootstrapDistTest
{
@ -63,13 +64,13 @@ namespace Tests.BasicTests
while (DateTime.UtcNow < endTime)
{
CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
CheckRoutingTables(GetAllOnlineCodexNodes());
if (DateTime.UtcNow > checkTime)
{
CheckRoutingTables(GetAllOnlineCodexNodes());
}
var node = RandomUtils.PickOneRandom(nodes.ToList());
var file = GenerateTestFile(50.MB());
node.UploadFile(file);
Thread.Sleep(5000);
Thread.Sleep(20000);
}
}