From 05e60a6b747cfbae7f75aac4525d9ff852beb167 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 3 Jul 2023 08:39:11 +0200 Subject: [PATCH] Future-counting test --- ContinuousTests/SingleTestRun.cs | 14 +- ContinuousTests/Tests/MarketplaceTest.cs | 164 ++++++++++----------- ContinuousTests/Tests/PerformanceTests.cs | 136 ++++++++--------- ContinuousTests/Tests/TransientNodeTest.cs | 87 ++++++----- ContinuousTests/Tests/TwoClientTest.cs | 4 +- DistTestCore/Codex/CodexAccess.cs | 5 + DistTestCore/Codex/CodexApiTypes.cs | 5 + 7 files changed, 221 insertions(+), 194 deletions(-) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index cb1c801..9316789 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -62,6 +62,7 @@ namespace ContinuousTests { try { + OverviewLog(" > Starting test. " + FuturesInfo()); RunTestMoments(); if (!config.KeepPassedTestLogs) fixtureLog.Delete(); @@ -112,7 +113,7 @@ namespace ContinuousTests { ThrowFailTest(); } - OverviewLog(" > Test passed."); + OverviewLog(" > Test passed. " + FuturesInfo()); return; } } @@ -122,10 +123,19 @@ namespace ContinuousTests { var ex = UnpackException(exceptions.First()); Log(ex.ToString()); - OverviewLog(" > Test failed: " + ex.Message); + OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message); throw ex; } + private string FuturesInfo() + { + var containers = config.CodexDeployment.CodexContainers; + var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet); + var f = nodes.Select(n => n.GetDebugFutures().ToString()); + var msg = $"(Futures: [{string.Join(", ", f)}])"; + return msg; + } + private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 02c3dcc..86500a2 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -1,99 +1,99 @@ -using DistTestCore; -using DistTestCore.Codex; -using Newtonsoft.Json; -using NUnit.Framework; -using Utils; +//using DistTestCore; +//using DistTestCore.Codex; +//using Newtonsoft.Json; +//using NUnit.Framework; +//using Utils; -namespace ContinuousTests.Tests -{ - public class MarketplaceTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 1; - public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(15); - public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - public override int EthereumAccountIndex => 200; - public override string CustomK8sNamespace => "codex-continuous-marketplace"; +//namespace ContinuousTests.Tests +//{ +// public class MarketplaceTest : ContinuousTest +// { +// public override int RequiredNumberOfNodes => 1; +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); +// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; +// public override int EthereumAccountIndex => 200; +// public override string CustomK8sNamespace => "codex-continuous-marketplace"; - private readonly uint numberOfSlots = 3; - private readonly ByteSize fileSize = 10.MB(); - private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); +// private readonly uint numberOfSlots = 3; +// private readonly ByteSize fileSize = 10.MB(); +// private readonly TestToken pricePerSlotPerSecond = 10.TestTokens(); - private TestFile file = null!; - private ContentId? cid; - private string purchaseId = string.Empty; +// private TestFile file = null!; +// private ContentId? cid; +// private string purchaseId = string.Empty; - [TestMoment(t: Zero)] - public void NodePostsStorageRequest() - { - var contractDuration = TimeSpan.FromMinutes(11); //TimeSpan.FromDays(3) + TimeSpan.FromHours(1); - decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); - var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000; +// [TestMoment(t: Zero)] +// public void NodePostsStorageRequest() +// { +// var contractDuration = TimeSpan.FromMinutes(8); +// decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds); +// var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000; - file = FileManager.GenerateTestFile(fileSize); +// file = FileManager.GenerateTestFile(fileSize); - NodeRunner.RunNode((codexAccess, marketplaceAccess) => - { - cid = UploadFile(codexAccess.Node, file); - Assert.That(cid, Is.Not.Null); +// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// { +// cid = UploadFile(codexAccess.Node, file); +// Assert.That(cid, Is.Not.Null); - purchaseId = marketplaceAccess.RequestStorage( - contentId: cid!, - pricePerSlotPerSecond: pricePerSlotPerSecond, - requiredCollateral: 100.TestTokens(), - minRequiredNumberOfNodes: numberOfSlots, - proofProbability: 10, - duration: contractDuration); +// purchaseId = marketplaceAccess.RequestStorage( +// contentId: cid!, +// pricePerSlotPerSecond: pricePerSlotPerSecond, +// requiredCollateral: 100.TestTokens(), +// minRequiredNumberOfNodes: numberOfSlots, +// proofProbability: 10, +// duration: contractDuration); - Assert.That(!string.IsNullOrEmpty(purchaseId)); +// Assert.That(!string.IsNullOrEmpty(purchaseId)); - WaitForContractToStart(codexAccess, purchaseId); - }); - } +// WaitForContractToStart(codexAccess, purchaseId); +// }); +// } - [TestMoment(t: MinuteFive * 2)] - public void StoredDataIsAvailableAfterThreeDays() - { - NodeRunner.RunNode((codexAccess, marketplaceAccess) => - { - var result = DownloadFile(codexAccess.Node, cid!); +// [TestMoment(t: MinuteFive + MinuteOne)] +// public void StoredDataIsAvailableAfterThreeDays() +// { +// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// { +// var result = DownloadFile(codexAccess.Node, cid!); - file.AssertIsEqual(result); - }); - } +// file.AssertIsEqual(result); +// }); +// } - private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) - { - var lastState = ""; - var waitStart = DateTime.UtcNow; - var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); - var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); +// private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId) +// { +// var lastState = ""; +// var waitStart = DateTime.UtcNow; +// var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024); +// var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); - Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); - while (lastState != "started") - { - CancelToken.ThrowIfCancellationRequested(); +// Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}"); +// while (lastState != "started") +// { +// CancelToken.ThrowIfCancellationRequested(); - var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); - var statusJson = JsonConvert.SerializeObject(purchaseStatus); - if (purchaseStatus != null && purchaseStatus.state != lastState) - { - lastState = purchaseStatus.state; - Log.Log("Purchase status: " + statusJson); - } +// var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); +// var statusJson = JsonConvert.SerializeObject(purchaseStatus); +// if (purchaseStatus != null && purchaseStatus.state != lastState) +// { +// lastState = purchaseStatus.state; +// Log.Log("Purchase status: " + statusJson); +// } - Thread.Sleep(2000); +// Thread.Sleep(2000); - if (lastState == "errored") - { - Assert.Fail("Contract start failed: " + statusJson); - } +// if (lastState == "errored") +// { +// Assert.Fail("Contract start failed: " + statusJson); +// } - if (DateTime.UtcNow - waitStart > maxWaitTime) - { - Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); - } - } - Log.Log("Contract started."); - } - } -} +// if (DateTime.UtcNow - waitStart > maxWaitTime) +// { +// Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}"); +// } +// } +// Log.Log("Contract started."); +// } +// } +//} diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index 2df3052..b6094a0 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -1,86 +1,86 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; +//using DistTestCore; +//using DistTestCore.Codex; +//using NUnit.Framework; -namespace ContinuousTests.Tests -{ - public class UploadPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; +//namespace ContinuousTests.Tests +//{ +// public class UploadPerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 1; - [TestMoment(t: Zero)] - public void UploadTest() - { - UploadTest(100, Nodes[0]); - } - } +// [TestMoment(t: Zero)] +// public void UploadTest() +// { +// UploadTest(100, Nodes[0]); +// } +// } - public class DownloadLocalPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; +// public class DownloadLocalPerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 1; - [TestMoment(t: Zero)] - public void DownloadTest() - { - DownloadTest(100, Nodes[0], Nodes[0]); - } - } +// [TestMoment(t: Zero)] +// public void DownloadTest() +// { +// DownloadTest(100, Nodes[0], Nodes[0]); +// } +// } - public class DownloadRemotePerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 2; +// public class DownloadRemotePerformanceTest : PerformanceTest +// { +// public override int RequiredNumberOfNodes => 2; - [TestMoment(t: Zero)] - public void DownloadTest() - { - DownloadTest(100, Nodes[0], Nodes[1]); - } - } +// [TestMoment(t: Zero)] +// public void DownloadTest() +// { +// DownloadTest(100, Nodes[0], Nodes[1]); +// } +// } - public abstract class PerformanceTest : ContinuousTest - { - public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); - public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; +// public abstract class PerformanceTest : ContinuousTest +// { +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); +// public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; - public void UploadTest(int megabytes, CodexAccess uploadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); +// public void UploadTest(int megabytes, CodexAccess uploadNode) +// { +// var file = FileManager.GenerateTestFile(megabytes.MB()); - var time = Measure(() => - { - UploadFile(uploadNode, file); - }); +// var time = Measure(() => +// { +// UploadFile(uploadNode, file); +// }); - var timePerMB = time / megabytes; +// var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); - } +// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); +// } - public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); +// public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) +// { +// var file = FileManager.GenerateTestFile(megabytes.MB()); - var cid = UploadFile(uploadNode, file); - Assert.That(cid, Is.Not.Null); +// var cid = UploadFile(uploadNode, file); +// Assert.That(cid, Is.Not.Null); - TestFile? result = null; - var time = Measure(() => - { - result = DownloadFile(downloadNode, cid!); - }); +// TestFile? result = null; +// var time = Measure(() => +// { +// result = DownloadFile(downloadNode, cid!); +// }); - file.AssertIsEqual(result); +// file.AssertIsEqual(result); - var timePerMB = time / megabytes; +// var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); - } +// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); +// } - private static TimeSpan Measure(Action action) - { - var start = DateTime.UtcNow; - action(); - return DateTime.UtcNow - start; - } - } -} +// private static TimeSpan Measure(Action action) +// { +// var start = DateTime.UtcNow; +// action(); +// return DateTime.UtcNow - start; +// } +// } +//} diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs index 37eb194..fe2ae34 100644 --- a/ContinuousTests/Tests/TransientNodeTest.cs +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -1,47 +1,54 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; +//using DistTestCore; +//using DistTestCore.Codex; +//using NUnit.Framework; -namespace ContinuousTests.Tests -{ - public class TransientNodeTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 3; - public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); - public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant(); - public override int EthereumAccountIndex => 201; +//namespace ContinuousTests.Tests +//{ +// public class TransientNodeTest : ContinuousTest +// { +// public override int RequiredNumberOfNodes => 3; +// public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1); +// public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; +// public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant(); +// public override int EthereumAccountIndex => 201; - private TestFile file = null!; - private ContentId cid = null!; +// private TestFile file = null!; +// private ContentId cid = null!; - private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } - private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } - private CodexAccess IntermediateNode { get { return Nodes[2]; } } +// private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } +// private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } +// private CodexAccess IntermediateNode { get { return Nodes[2]; } } - [TestMoment(t: 0)] - public void UploadWithTransientNode() - { - file = FileManager.GenerateTestFile(10.MB()); +// [TestMoment(t: 0)] +// public void UploadWithTransientNode() +// { +// file = FileManager.GenerateTestFile(10.MB()); - NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess) => - { - cid = UploadFile(codexAccess, file)!; - Assert.That(cid, Is.Not.Null); +// NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// { +// cid = UploadFile(codexAccess, file)!; +// Assert.That(cid, Is.Not.Null); - var resultFile = DownloadFile(IntermediateNode, cid); - file.AssertIsEqual(resultFile); - }); - } +// var dlt = Task.Run(() => +// { +// Thread.Sleep(10000); +// lifecycle.DownloadLog(codexAccess.Container); +// }); - [TestMoment(t: MinuteFive)] - public void DownloadWithTransientNode() - { - NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => - { - var resultFile = DownloadFile(codexAccess, cid); - file.AssertIsEqual(resultFile); - }); - } - } -} +// var resultFile = DownloadFile(IntermediateNode, cid); +// dlt.Wait(); +// file.AssertIsEqual(resultFile); +// }); +// } + +// [TestMoment(t: 30)] +// public void DownloadWithTransientNode() +// { +// NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// { +// var resultFile = DownloadFile(codexAccess, cid); +// file.AssertIsEqual(resultFile); +// }); +// } +// } +//} diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index 9d23d77..53a26ba 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -6,7 +6,7 @@ namespace ContinuousTests.Tests public class TwoClientTest : ContinuousTest { public override int RequiredNumberOfNodes => 2; - public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; private ContentId? cid; @@ -21,7 +21,7 @@ namespace ContinuousTests.Tests Assert.That(cid, Is.Not.Null); } - [TestMoment(t: MinuteFive)] + [TestMoment(t: 10)] public void DownloadTestFile() { var dl = DownloadFile(Nodes[1], cid!); diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 2aea3b6..0ec046c 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -48,6 +48,11 @@ namespace DistTestCore.Codex return result; } + public int GetDebugFutures() + { + return Http().HttpGetJson("debug/futures").futures; + } + public string UploadFile(FileStream fileStream) { return Http().HttpPostStream("upload", fileStream); diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index 885540e..835df10 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -16,6 +16,11 @@ namespace DistTestCore.Codex public CodexDebugTableResponse table { get; set; } = new(); } + public class CodexDebugFutures + { + public int futures { get; set; } + } + public class CodexDebugTableResponse { public CodexDebugTableNodeResponse localNode { get; set; } = new();