From 7eb3bbd1deeb4cb6bd7d95561b4ef1294d0696a2 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Wed, 23 Apr 2025 10:22:23 +0200 Subject: [PATCH 1/7] Setup example in lifecycelytest --- Tests/CodexReleaseTests/Parallelism.cs | 2 +- Tests/FrameworkTests/LifecycelyTest.cs | 226 +++++++++++++++++++++++++ 2 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 Tests/FrameworkTests/LifecycelyTest.cs diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs index a1b26c73..f6e68cc0 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/CodexReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; [assembly: LevelOfParallelism(1)] -namespace CodexReleaseTests.DataTests +namespace CodexReleaseTests { } diff --git a/Tests/FrameworkTests/LifecycelyTest.cs b/Tests/FrameworkTests/LifecycelyTest.cs new file mode 100644 index 00000000..ef90a48c --- /dev/null +++ b/Tests/FrameworkTests/LifecycelyTest.cs @@ -0,0 +1,226 @@ +using NUnit.Framework; + +namespace FrameworkTests +{ + [Parallelizable(ParallelScope.All)] + [TestFixture(10)] + [TestFixture(20)] + [TestFixture(30)] + public class LifecycelyTest + { + public LifecycelyTest(int num) + { + Log("ctor", GetCurrentTestName(), num); + this.num = num; + } + + [SetUp] + public void Setup() + { + Log(nameof(Setup), GetCurrentTestName()); + } + + [TearDown] + public void TearDown() + { + Log(nameof(TearDown), GetCurrentTestName()); + } + + [Test] + public void A() + { + Log(nameof(A), "Run"); + SleepRandom(); + Log(nameof(A), "Finish"); + } + + [Test] + public void B() + { + Log(nameof(B), "Run"); + SleepRandom(); + Log(nameof(B), "Finish"); + } + + [Test] + public void C() + { + Log(nameof(C), "Run"); + SleepRandom(); + Log(nameof(C), "Finish"); + } + + [Test] + [Combinatorial] + public void Multi( + [Values(1, 2, 3)] int num) + { + Log(nameof(Multi), "Run", num); + SleepRandom(); + Log(nameof(Multi), "Finish", num); + } + + + + + + + + + + + + + private static readonly Random r = new Random(); + private readonly int num; + + private void SleepRandom() + { + Thread.Sleep(TimeSpan.FromSeconds(5.0)); + Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); + } + + private void Log(string scope, string msg) + { + ALog.Log($"{num} {scope} {msg}"); + } + + private void Log(string scope, string msg, int num) + { + ALog.Log($"{this.num} {scope} {msg} {num}"); + } + + private string GetCurrentTestName() + { + return $"[{TestContext.CurrentContext.Test.Name}]"; + } + } + + + + + public class ALog + { + private static readonly object _lock = new object(); + + public static void Log(string msg) + { + lock (_lock) + { + File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); + } + } + } + + + + + + + + + + + + public interface ITestLifecycleComponent + { + void Start(); + void Stop(string results); + } + + + public class Base + { + private readonly Dictionary> anyFields = new(); + + public void Setup() + { + var testId = 23; + + var fields = new Dictionary(); + anyFields.Add(testId, fields); + YieldFields(field => + { + fields.Add(field.GetType(), field); + }); + + foreach (var field in fields.Values) + { + field.Start(); + } + } + + public void TearDown() + { + var testId = 23; + + // foreach stop + + anyFields.Remove(testId); + } + + public T Get() + { + int testId = 123; + var fields = anyFields[testId]; + var type = typeof(T); + var result = fields[type]; + return (T)result; + } + + public BaseFields GetBaseField() + { + return Get(); + } + + protected virtual void YieldFields(Action giveField) + { + giveField(new BaseFields()); + } + } + + public class Mid : Base + { + protected override void YieldFields(Action giveField) + { + base.YieldFields(giveField); + giveField(new MidFields()); + } + + public MidFields GetMid() + { + return Get(); + } + } + + public class Top : Mid + { + protected override void YieldFields(Action giveField) + { + base.YieldFields(giveField); + giveField(new TopFields()); + } + + public TopFields GetTop() + { + return Get(); + } + } + + public class BaseFields : ITestLifecycleComponent + { + public string EntryPoint { get; set; } = string.Empty; + public string Log { get; set; } = string.Empty; + } + + public class MidFields : ITestLifecycleComponent + { + public string Nodes { get; set; } = string.Empty; + } + + public class TopFields : ITestLifecycleComponent + { + public string Geth { get; set; } = string.Empty; + public string Contracts { get; set; } = string.Empty; + } +} From 2dfdfac2bb52e4ff919ee5f1b3986a839d71b0a2 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Wed, 23 Apr 2025 14:18:11 +0200 Subject: [PATCH 2/7] while better the new plan is still a mess --- .../OverwatchSupport/CodexTranscriptWriter.cs | 15 +- .../CodexTranscriptWriterConfig.cs | 4 +- .../MarketplaceAutoBootstrapDistTest.cs | 39 +- Tests/DistTestCore/DistTest.cs | 124 ++----- .../DistTestLifecycleComponents.cs | 97 +++++ Tests/DistTestCore/Logs/FixtureLog.cs | 13 +- Tests/DistTestCore/Logs/TestLog.cs | 2 - Tests/DistTestCore/TestLifecycle.cs | 30 +- .../AutoBootstrapDistTest.cs | 58 +-- Tests/ExperimentalTests/CodexDistTest.cs | 215 ++++------- .../CodexLogTrackerProvider.cs | 73 ++++ Tests/FrameworkTests/LifecycelyTest.cs | 334 +++++++++--------- Tests/FrameworkTests/Parallelism.cs | 6 + 13 files changed, 549 insertions(+), 461 deletions(-) create mode 100644 Tests/DistTestCore/DistTestLifecycleComponents.cs create mode 100644 Tests/ExperimentalTests/CodexLogTrackerProvider.cs create mode 100644 Tests/FrameworkTests/Parallelism.cs diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index afd8d1a2..53ac1496 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -23,16 +23,27 @@ namespace CodexPlugin.OverwatchSupport converter = new CodexLogConverter(writer, config, identityMap); } - public void Finalize(string outputFilepath) + public void Finalize() { log.Log("Finalizing Codex transcript..."); writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); - writer.Write(outputFilepath); + writer.Write(GetOutputFullPath()); log.Log("Done"); } + private string GetOutputFullPath() + { + var outputPath = Path.GetDirectoryName(log.GetFullName()); + if (outputPath == null) throw new Exception("Logfile path is null"); + var filename = Path.GetFileNameWithoutExtension(log.GetFullName()); + if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); + var outputFile = Path.Combine(outputPath, filename + "_" + config.OutputFilename); + if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; + return outputFile; + } + public ICodexNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs index 247494c8..8c5e7bf0 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs @@ -2,11 +2,13 @@ { public class CodexTranscriptWriterConfig { - public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents) + public CodexTranscriptWriterConfig(string outputFilename, bool includeBlockReceivedEvents) { + OutputFilename = outputFilename; IncludeBlockReceivedEvents = includeBlockReceivedEvents; } + public string OutputFilename { get; } public bool IncludeBlockReceivedEvents { get; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index a5ec0885..f6e8a9ca 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -10,18 +10,41 @@ using Utils; namespace CodexReleaseTests.MarketTests { + public class MarketplaceTestComponent : ILifecycleComponent + { + public IGethNode Geth { get; } + public ICodexContracts Contracts { get; } + + public void Start(ILifecycleComponentAccess access) + { + throw new NotImplementedException(); + } + + public void Stop(ILifecycleComponentAccess access, DistTestResult result) + { + throw new NotImplementedException(); + } + } + public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest { - private readonly Dictionary handles = new Dictionary(); protected const int StartingBalanceTST = 1000; protected const int StartingBalanceEth = 10; + protected override void CreateComponents(ILifecycleComponentCollector collector) + { + base.CreateComponents(collector); + + + collector.AddComponent(new MarketplaceTestComponent()); + } + protected override void LifecycleStart(TestLifecycle lifecycle) { base.LifecycleStart(lifecycle); var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - handles.Add(lifecycle, new MarketplaceHandle(geth, contracts)); + handles.Add(lifecycle, new MarketplaceTestComponent(geth, contracts)); } protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) @@ -323,17 +346,5 @@ namespace CodexReleaseTests.MarketTests public SlotFilledEventDTO SlotFilledEvent { get; } public ICodexNode Host { get; } } - - private class MarketplaceHandle - { - public MarketplaceHandle(IGethNode geth, ICodexContracts contracts) - { - Geth = geth; - Contracts = contracts; - } - - public IGethNode Geth { get; } - public ICodexContracts Contracts { get; } - } } } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 534eb74a..9a969532 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -19,9 +19,8 @@ namespace DistTestCore private readonly Assembly[] testAssemblies; private readonly FixtureLog fixtureLog; private readonly StatusLog statusLog; - private readonly object lifecycleLock = new object(); private readonly EntryPoint globalEntryPoint; - private readonly Dictionary lifecycles = new Dictionary(); + private readonly DistTestLifecycleComponents lifecycleComponents = new DistTestLifecycleComponents(); private readonly string deployId; public DistTest() @@ -89,7 +88,12 @@ namespace DistTestCore { try { - CreateNewTestLifecycle(); + var testName = GetCurrentTestName(); + fixtureLog.WriteLogTag(); + Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => + { + lifecycleComponents.Setup(testName, CreateComponents); + }); } catch (Exception ex) { @@ -104,7 +108,7 @@ namespace DistTestCore { try { - DisposeTestLifecycle(); + Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", DisposeTestLifecycle); } catch (Exception ex) { @@ -171,80 +175,49 @@ namespace DistTestCore { } - protected virtual void LifecycleStart(TestLifecycle lifecycle) + protected virtual void CreateComponents(ILifecycleComponentCollector collector) + { + var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); + var lifecycle = new TestLifecycle( + fixtureLog.CreateTestLog(), + configuration, + GetWebCallTimeSet(), + GetK8sTimeSet(), + testNamespace, + GetCurrentTestName(), + deployId, + ShouldWaitForCleanup()); + + collector.AddComponent(lifecycle); + } + + protected virtual void DestroyComponents(TestLifecycle lifecycle, DistTestResult testResult) { } - protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult) + public T Get() where T : ILifecycleComponent { + return lifecycleComponents.Get(GetCurrentTestName()); } - protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) + private TestLifecycle Get() { - } - - protected TestLifecycle Get() - { - lock (lifecycleLock) - { - return lifecycles[GetCurrentTestName()]; - } - } - - private void CreateNewTestLifecycle() - { - var testName = GetCurrentTestName(); - fixtureLog.WriteLogTag(); - Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => - { - lock (lifecycleLock) - { - var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); - var lifecycle = new TestLifecycle( - fixtureLog.CreateTestLog(), - configuration, - GetWebCallTimeSet(), - GetK8sTimeSet(), - testNamespace, - deployId, - ShouldWaitForCleanup()); - lifecycles.Add(testName, lifecycle); - LifecycleStart(lifecycle); - } - }); + return Get(); } private void DisposeTestLifecycle() { + var testName = GetCurrentTestName(); + var results = GetTestResult(); var lifecycle = Get(); - var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); var data = lifecycle.GetPluginMetadata(); - CollectStatusLogData(lifecycle, data); - fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration, data); - Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => - { - WriteEndTestLog(lifecycle.Log); + fixtureLog.Log($"{GetCurrentTestName()} = {results} ({testDuration})"); + statusLog.ConcludeTest(results, testDuration, data); - IncludeLogsOnTestFailure(lifecycle); - LifecycleStop(lifecycle, testResult); - lifecycle.DeleteAllResources(); - lifecycles.Remove(GetCurrentTestName()); - }); + lifecycleComponents.TearDown(testName, results); } - private void WriteEndTestLog(TestLog log) - { - var result = TestContext.CurrentContext.Result; - - Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}"); - if (!string.IsNullOrEmpty(result.Message)) - { - Log(result.Message); - Log($"{result.StackTrace}"); - } - } private IWebCallTimeSet GetWebCallTimeSet() { @@ -296,28 +269,6 @@ namespace DistTestCore .ToArray(); } - private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) - { - var testStatus = TestContext.CurrentContext.Result.Outcome.Status; - if (ShouldDownloadAllLogs(testStatus)) - { - lifecycle.Log.Log("Downloading all container logs..."); - lifecycle.DownloadAllLogs(); - } - } - - private bool ShouldDownloadAllLogs(TestStatus testStatus) - { - if (configuration.AlwaysDownloadContainerLogs) return true; - if (!IsDownloadingLogsEnabled()) return false; - if (testStatus == TestStatus.Failed) - { - return true; - } - - return false; - } - private string GetCurrentTestName() { return $"[{TestContext.CurrentContext.Test.Name}]"; @@ -328,7 +279,8 @@ namespace DistTestCore var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed; var status = TestContext.CurrentContext.Result.Outcome.Status.ToString(); var result = TestContext.CurrentContext.Result.Message; - return new DistTestResult(success, status, result ?? string.Empty); + var trace = TestContext.CurrentContext.Result.StackTrace; + return new DistTestResult(success, status, result ?? string.Empty, trace ?? string.Empty); } private bool IsDownloadingLogsEnabled() @@ -339,16 +291,18 @@ namespace DistTestCore public class DistTestResult { - public DistTestResult(bool success, string status, string result) + public DistTestResult(bool success, string status, string result, string trace) { Success = success; Status = status; Result = result; + Trace = trace; } public bool Success { get; } public string Status { get; } public string Result { get; } + public string Trace { get; } public override string ToString() { diff --git a/Tests/DistTestCore/DistTestLifecycleComponents.cs b/Tests/DistTestCore/DistTestLifecycleComponents.cs new file mode 100644 index 00000000..203a891c --- /dev/null +++ b/Tests/DistTestCore/DistTestLifecycleComponents.cs @@ -0,0 +1,97 @@ +namespace DistTestCore +{ + public interface ILifecycleComponent + { + void Start(ILifecycleComponentAccess access); + void Stop(ILifecycleComponentAccess access, DistTestResult result); + } + + public interface ILifecycleComponentCollector + { + void AddComponent(ILifecycleComponent component); + } + + public interface ILifecycleComponentAccess + { + T Get() where T : ILifecycleComponent; + } + + public class DistTestLifecycleComponents + { + private readonly object _lock = new object(); + private readonly Dictionary> components = new(); + + public void Setup(string testName, Action initializer) + { + var newComponents = new Dictionary(); + lock (_lock) + { + components.Add(testName, newComponents); + var collector = new Collector(newComponents); + initializer(collector); + } + + var access = new ScopedAccess(this, testName); + foreach (var component in newComponents.Values) + { + component.Start(access); + } + } + + public T Get(string testName) where T : ILifecycleComponent + { + var type = typeof(T); + lock (_lock) + { + return (T)components[testName][type]; + } + } + + public void TearDown(string testName, DistTestResult result) + { + var access = new ScopedAccess(this, testName); + var closingComponents = components[testName]; + foreach (var component in closingComponents.Values) + { + component.Stop(access, result); + } + + lock (_lock) + { + components.Remove(testName); + } + } + + private class Collector : ILifecycleComponentCollector + { + private readonly Dictionary components; + + public Collector(Dictionary components) + { + this.components = components; + } + + public void AddComponent(ILifecycleComponent component) + { + components.Add(component.GetType(), component); + } + } + + private class ScopedAccess : ILifecycleComponentAccess + { + private readonly DistTestLifecycleComponents parent; + private readonly string testName; + + public ScopedAccess(DistTestLifecycleComponents parent, string testName) + { + this.parent = parent; + this.testName = testName; + } + + public T Get() where T : ILifecycleComponent + { + return parent.Get(testName); + } + } + } +} diff --git a/Tests/DistTestCore/Logs/FixtureLog.cs b/Tests/DistTestCore/Logs/FixtureLog.cs index 9d3c77d3..f90147d8 100644 --- a/Tests/DistTestCore/Logs/FixtureLog.cs +++ b/Tests/DistTestCore/Logs/FixtureLog.cs @@ -4,26 +4,25 @@ namespace DistTestCore.Logs { public class FixtureLog : BaseTestLog { - private readonly ILog backingLog; - private readonly string deployId; - public FixtureLog(ILog backingLog, string deployId) : base(backingLog, deployId) { - this.backingLog = backingLog; - this.deployId = deployId; } public TestLog CreateTestLog(string name = "") { - return TestLog.Create(this, name); + var result = TestLog.Create(this, name); + result.Log(NameUtils.GetRawFixtureName()); + return result; } public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "") { var fullName = NameUtils.GetFixtureFullName(config, start, name); var log = CreateMainLog(fullName, name); - return new FixtureLog(log, deployId); + var result = new FixtureLog(log, deployId); + result.Log(NameUtils.GetRawFixtureName()); + return result; } } } diff --git a/Tests/DistTestCore/Logs/TestLog.cs b/Tests/DistTestCore/Logs/TestLog.cs index 0dca1464..6a334138 100644 --- a/Tests/DistTestCore/Logs/TestLog.cs +++ b/Tests/DistTestCore/Logs/TestLog.cs @@ -1,5 +1,4 @@ using Logging; -using System.Xml.Linq; namespace DistTestCore.Logs { @@ -8,7 +7,6 @@ namespace DistTestCore.Logs public TestLog(ILog backingLog, string methodName, string deployId, string name = "") : base(backingLog, deployId) { - backingLog.Log($"*** Begin: {methodName}"); } public static TestLog Create(FixtureLog parentLog, string name = "") diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 3d642d20..550b6151 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -10,22 +10,24 @@ using WebUtils; namespace DistTestCore { - public class TestLifecycle : IK8sHooks + public class TestLifecycle : IK8sHooks, ILifecycleComponent { private const string TestsType = "dist-tests"; private readonly EntryPoint entryPoint; private readonly Dictionary metadata; private readonly List runningContainers = new(); + private readonly string testName; private readonly string deployId; private readonly List stoppedContainerLogs = new List(); - public TestLifecycle(TestLog log, Configuration configuration, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8sTimeSet, string testNamespace, string deployId, bool waitForCleanup) + public TestLifecycle(TestLog log, Configuration configuration, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8sTimeSet, string testNamespace, string testName, string deployId, bool waitForCleanup) { Log = log; Configuration = configuration; WebCallTimeSet = webCallTimeSet; K8STimeSet = k8sTimeSet; TestNamespace = testNamespace; + this.testName = testName; TestStart = DateTime.UtcNow; entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(k8sTimeSet, this, testNamespace), configuration.GetFileManagerFolder(), webCallTimeSet, k8sTimeSet); @@ -36,6 +38,28 @@ namespace DistTestCore log.WriteLogTag(); } + public void Start(ILifecycleComponentAccess access) + { + Log.Log($"*** Begin: {testName}"); + } + + public void Stop(ILifecycleComponentAccess access, DistTestResult result) + { + Log.Log($"*** Finished: {testName} = {result.Status}"); + if (!string.IsNullOrEmpty(result.Result)) + { + Log.Log(result.Result); + Log.Log($"{result.Trace}"); + } + + if (!result.Success) + { + DownloadAllLogs(); + } + + DeleteAllResources(); + } + public DateTime TestStart { get; } public TestLog Log { get; } public Configuration Configuration { get; } @@ -45,7 +69,7 @@ namespace DistTestCore public bool WaitForCleanup { get; } public CoreInterface CoreInterface { get; } - public void DeleteAllResources() + private void DeleteAllResources() { entryPoint.Decommission( deleteKubernetesResources: true, diff --git a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs index d8ac6805..a2477597 100644 --- a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs +++ b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs @@ -4,46 +4,56 @@ using DistTestCore; namespace CodexTests { - public class AutoBootstrapDistTest : CodexDistTest + public class AutoBootstrapComponent : ILifecycleComponent { - private readonly Dictionary bootstrapNodes = new Dictionary(); - private bool isBooting = false; + public ICodexNode? BootstrapNode { get; private set; } = null; - protected override void LifecycleStart(TestLifecycle tl) + public void Start(ILifecycleComponentAccess access) { - base.LifecycleStart(tl); - if (!bootstrapNodes.ContainsKey(tl)) - { - isBooting = true; - bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace))); - isBooting = false; - } + if (BootstrapNode != null) return; + + var tl = access.Get(); + var ci = tl.CoreInterface; + var testNamespace = tl.TestNamespace; + + BootstrapNode = ci.StartCodexNode(s => s.WithName("BOOTSTRAP_" + testNamespace)); } - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) + public void ApplyBootstrapNode(ICodexSetup setup) { - bootstrapNodes.Remove(lifecycle); - base.LifecycleStop(lifecycle, result); + if (BootstrapNode == null) return; + + setup.WithBootstrapNode(BootstrapNode); + } + + public void Stop(ILifecycleComponentAccess access, DistTestResult result) + { + if (BootstrapNode == null) return; + BootstrapNode.Stop(waitTillStopped: false); + } + } + + public class AutoBootstrapDistTest : CodexDistTest + { + + protected override void CreateComponents(ILifecycleComponentCollector collector) + { + base.CreateComponents(collector); + collector.AddComponent(new AutoBootstrapComponent()); } protected override void OnCodexSetup(ICodexSetup setup) { - if (isBooting) return; - - var node = BootstrapNode; - if (node != null) setup.WithBootstrapNode(node); + Get().ApplyBootstrapNode(setup); } protected ICodexNode BootstrapNode { get { - var tl = Get(); - if (bootstrapNodes.TryGetValue(tl, out var node)) - { - return node; - } - throw new InvalidOperationException("Bootstrap node not yet started."); + var bn = Get().BootstrapNode; + if (bn == null) throw new InvalidOperationException("BootstrapNode accessed before initialized."); + return bn; } } } diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index d27216d3..40e711fc 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -1,6 +1,5 @@ using BlockchainUtils; using CodexClient; -using CodexClient.Hooks; using CodexContractsPlugin; using CodexNetDeployer; using CodexPlugin; @@ -17,86 +16,74 @@ using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using OverwatchTranscript; -using Utils; namespace CodexTests { - public class CodexLogTrackerProvider : ICodexHooksProvider + public class CodexDistTestComponents : ILifecycleComponent { - private readonly Action addNode; + private readonly object nodesLock = new object(); - public CodexLogTrackerProvider(Action addNode) + public CodexDistTestComponents(CodexTranscriptWriter? writer) { - this.addNode = addNode; + Writer = writer; } - // See TestLifecycle.cs DownloadAllLogs() - public ICodexNodeHooks CreateHooks(string nodeName) + public CodexTranscriptWriter? Writer { get; } + public BlockCache Cache { get; } = new(); + public List Nodes { get; } = new(); + + public void Start(ILifecycleComponentAccess access) { - return new CodexLogTracker(addNode); + var ci = access.Get().CoreInterface; + ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => + { + lock (nodesLock) + { + Nodes.Add(n); + } + })); } - public class CodexLogTracker : ICodexNodeHooks + public void Stop(ILifecycleComponentAccess access, DistTestResult result) { - private readonly Action addNode; + var tl = access.Get(); + var log = tl.Log; + var logFiles = tl.DownloadAllLogs(); - public CodexLogTracker(Action addNode) + TeardownTranscript(log, logFiles, result); + + // todo: on not success: go to nodes and dl logs? + // or fix disttest failure log download so we can always have logs even for non-codexes? + } + + private void TeardownTranscript(TestLog log, IDownloadedLog[] logFiles, DistTestResult result) + { + if (Writer == null) return; + + Writer.AddResult(result.Success, result.Result); + + try { - this.addNode = addNode; + Stopwatch.Measure(log, "Transcript.ProcessLogs", () => + { + Writer.ProcessLogs(logFiles); + }); + + Stopwatch.Measure(log, $"Transcript.Finalize", () => + { + Writer.IncludeFile(log.GetFullName()); + Writer.Finalize(); + }); } - - public void OnFileDownloaded(ByteSize size, ContentId cid) - { - } - - public void OnFileDownloading(ContentId cid) - { - } - - public void OnFileUploaded(string uid, ByteSize size, ContentId cid) - { - } - - public void OnFileUploading(string uid, ByteSize size) - { - } - - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) - { - addNode(node); - } - - public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) - { - } - - public void OnNodeStopping() - { - } - - public void OnStorageAvailabilityCreated(StorageAvailability response) - { - } - - public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) - { - } - - public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + catch (Exception ex) { + log.Error("Failure during transcript teardown: " + ex); } } } public class CodexDistTest : DistTest { - private static readonly object _lock = new object(); - private static readonly Dictionary writers = new Dictionary(); - private static readonly Dictionary blockCaches = new Dictionary(); - - // this entire structure is not good and needs to be destroyed at the earliest convenience: - private static readonly Dictionary> nodes = new Dictionary>(); - public CodexDistTest() { ProjectPlugin.Load(); @@ -112,34 +99,12 @@ namespace CodexTests localBuilder.Build(); } - protected override void LifecycleStart(TestLifecycle lifecycle) + protected override void CreateComponents(ILifecycleComponentCollector collector) { - base.LifecycleStart(lifecycle); - SetupTranscript(lifecycle); - - Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => - { - lock (_lock) - { - if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List()); - nodes[lifecycle].Add(n); - } - })); - } - - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) - { - base.LifecycleStop(lifecycle, result); - TeardownTranscript(lifecycle, result); - - if (!result.Success) - { - lock (_lock) - { - var codexNodes = nodes[lifecycle]; - foreach (var node in codexNodes) node.DownloadLog(); - } - } + base.CreateComponents(collector); + collector.AddComponent(new CodexDistTestComponents( + SetupTranscript() + )); } public ICodexNode StartCodex() @@ -173,6 +138,11 @@ namespace CodexTests return Ci.StartGethNode(GetBlockCache(), setup); } + private BlockCache GetBlockCache() + { + return Get().Cache; + } + public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() { return new PeerConnectionTestHelpers(GetTestLog()); @@ -180,7 +150,7 @@ namespace CodexTests public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() { - return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); + return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); } public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") @@ -258,81 +228,20 @@ namespace CodexTests return null; } - private void SetupTranscript(TestLifecycle lifecycle) + private CodexTranscriptWriter? SetupTranscript() { var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; + if (attr == null) return null; var config = new CodexTranscriptWriterConfig( + attr.OutputFilename, attr.IncludeBlockReceivedEvents ); - var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); + var log = new LogPrefixer(GetTestLog(), "(Transcript) "); var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); Ci.AddCodexHooksProvider(writer); - lock (_lock) - { - writers.Add(lifecycle, writer); - } - } - - private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result) - { - var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; - - var outputFilepath = GetOutputFullPath(lifecycle, attr); - - CodexTranscriptWriter writer = null!; - lock (_lock) - { - writer = writers[lifecycle]; - writers.Remove(lifecycle); - } - - writer.AddResult(result.Success, result.Result); - - try - { - Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () => - { - writer.ProcessLogs(lifecycle.DownloadAllLogs()); - }); - - Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () => - { - writer.IncludeFile(lifecycle.Log.GetFullName()); - writer.Finalize(outputFilepath); - }); - } - catch (Exception ex) - { - lifecycle.Log.Error("Failure during transcript teardown: " + ex); - } - } - - private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr) - { - var outputPath = Path.GetDirectoryName(lifecycle.Log.GetFullName()); - if (outputPath == null) throw new Exception("Logfile path is null"); - var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.GetFullName()); - if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); - var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename); - if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; - return outputFile; - } - - private BlockCache GetBlockCache() - { - var lifecycle = Get(); - lock (_lock) - { - if (!blockCaches.ContainsKey(lifecycle)) - { - blockCaches[lifecycle] = new BlockCache(); - } - } - return blockCaches[lifecycle]; + return writer; } } diff --git a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs new file mode 100644 index 00000000..1b2fdd66 --- /dev/null +++ b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs @@ -0,0 +1,73 @@ +using CodexClient; +using CodexClient.Hooks; +using Utils; + +namespace CodexTests +{ + public class CodexLogTrackerProvider : ICodexHooksProvider + { + private readonly Action addNode; + + public CodexLogTrackerProvider(Action addNode) + { + this.addNode = addNode; + } + + // See TestLifecycle.cs DownloadAllLogs() + public ICodexNodeHooks CreateHooks(string nodeName) + { + return new CodexLogTracker(addNode); + } + + public class CodexLogTracker : ICodexNodeHooks + { + private readonly Action addNode; + + public CodexLogTracker(Action addNode) + { + this.addNode = addNode; + } + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + addNode(node); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + } + + public void OnNodeStopping() + { + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + { + } + } + } +} diff --git a/Tests/FrameworkTests/LifecycelyTest.cs b/Tests/FrameworkTests/LifecycelyTest.cs index ef90a48c..6c11e14f 100644 --- a/Tests/FrameworkTests/LifecycelyTest.cs +++ b/Tests/FrameworkTests/LifecycelyTest.cs @@ -1,64 +1,64 @@ -using NUnit.Framework; +//using NUnit.Framework; -namespace FrameworkTests -{ - [Parallelizable(ParallelScope.All)] - [TestFixture(10)] - [TestFixture(20)] - [TestFixture(30)] - public class LifecycelyTest - { - public LifecycelyTest(int num) - { - Log("ctor", GetCurrentTestName(), num); - this.num = num; - } +//namespace FrameworkTests +//{ +// [Parallelizable(ParallelScope.All)] +// [TestFixture(10)] +// [TestFixture(20)] +// [TestFixture(30)] +// public class LifecycelyTest +// { +// public LifecycelyTest(int num) +// { +// Log("ctor", GetCurrentTestName(), num); +// this.num = num; +// } - [SetUp] - public void Setup() - { - Log(nameof(Setup), GetCurrentTestName()); - } +// [SetUp] +// public void Setup() +// { +// Log(nameof(Setup), GetCurrentTestName()); +// } - [TearDown] - public void TearDown() - { - Log(nameof(TearDown), GetCurrentTestName()); - } +// [TearDown] +// public void TearDown() +// { +// Log(nameof(TearDown), GetCurrentTestName()); +// } - [Test] - public void A() - { - Log(nameof(A), "Run"); - SleepRandom(); - Log(nameof(A), "Finish"); - } +// [Test] +// public void A() +// { +// Log(nameof(A), "Run"); +// SleepRandom(); +// Log(nameof(A), "Finish"); +// } - [Test] - public void B() - { - Log(nameof(B), "Run"); - SleepRandom(); - Log(nameof(B), "Finish"); - } +// [Test] +// public void B() +// { +// Log(nameof(B), "Run"); +// SleepRandom(); +// Log(nameof(B), "Finish"); +// } - [Test] - public void C() - { - Log(nameof(C), "Run"); - SleepRandom(); - Log(nameof(C), "Finish"); - } +// [Test] +// public void C() +// { +// Log(nameof(C), "Run"); +// SleepRandom(); +// Log(nameof(C), "Finish"); +// } - [Test] - [Combinatorial] - public void Multi( - [Values(1, 2, 3)] int num) - { - Log(nameof(Multi), "Run", num); - SleepRandom(); - Log(nameof(Multi), "Finish", num); - } +// [Test] +// [Combinatorial] +// public void Multi( +// [Values(1, 2, 3)] int num) +// { +// Log(nameof(Multi), "Run", num); +// SleepRandom(); +// Log(nameof(Multi), "Finish", num); +// } @@ -71,46 +71,46 @@ namespace FrameworkTests - private static readonly Random r = new Random(); - private readonly int num; +// private static readonly Random r = new Random(); +// private readonly int num; - private void SleepRandom() - { - Thread.Sleep(TimeSpan.FromSeconds(5.0)); - Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); - } +// private void SleepRandom() +// { +// Thread.Sleep(TimeSpan.FromSeconds(5.0)); +// Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); +// } - private void Log(string scope, string msg) - { - ALog.Log($"{num} {scope} {msg}"); - } +// private void Log(string scope, string msg) +// { +// ALog.Log($"{num} {scope} {msg}"); +// } - private void Log(string scope, string msg, int num) - { - ALog.Log($"{this.num} {scope} {msg} {num}"); - } +// private void Log(string scope, string msg, int num) +// { +// ALog.Log($"{this.num} {scope} {msg} {num}"); +// } - private string GetCurrentTestName() - { - return $"[{TestContext.CurrentContext.Test.Name}]"; - } - } +// private string GetCurrentTestName() +// { +// return $"[{TestContext.CurrentContext.Test.Name}]"; +// } +// } - public class ALog - { - private static readonly object _lock = new object(); +// public class ALog +// { +// private static readonly object _lock = new object(); - public static void Log(string msg) - { - lock (_lock) - { - File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); - } - } - } +// public static void Log(string msg) +// { +// lock (_lock) +// { +// File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); +// } +// } +// } @@ -122,105 +122,99 @@ namespace FrameworkTests - public interface ITestLifecycleComponent - { - void Start(); - void Stop(string results); - } +// public class Base +// { +// private readonly Dictionary> anyFields = new(); - public class Base - { - private readonly Dictionary> anyFields = new(); +// public void Setup() +// { +// var testId = 23; - public void Setup() - { - var testId = 23; +// var fields = new Dictionary(); +// anyFields.Add(testId, fields); +// YieldFields(field => +// { +// fields.Add(field.GetType(), field); +// }); - var fields = new Dictionary(); - anyFields.Add(testId, fields); - YieldFields(field => - { - fields.Add(field.GetType(), field); - }); +// foreach (var field in fields.Values) +// { +// field.Start(); +// } +// } - foreach (var field in fields.Values) - { - field.Start(); - } - } +// public void TearDown() +// { +// var testId = 23; - public void TearDown() - { - var testId = 23; +// // foreach stop - // foreach stop +// anyFields.Remove(testId); +// } - anyFields.Remove(testId); - } +// public T Get() +// { +// int testId = 123; +// var fields = anyFields[testId]; +// var type = typeof(T); +// var result = fields[type]; +// return (T)result; +// } - public T Get() - { - int testId = 123; - var fields = anyFields[testId]; - var type = typeof(T); - var result = fields[type]; - return (T)result; - } +// public BaseFields GetBaseField() +// { +// return Get(); +// } - public BaseFields GetBaseField() - { - return Get(); - } +// protected virtual void YieldFields(Action giveField) +// { +// giveField(new BaseFields()); +// } +// } - protected virtual void YieldFields(Action giveField) - { - giveField(new BaseFields()); - } - } +// public class Mid : Base +// { +// protected override void YieldFields(Action giveField) +// { +// base.YieldFields(giveField); +// giveField(new MidFields()); +// } - public class Mid : Base - { - protected override void YieldFields(Action giveField) - { - base.YieldFields(giveField); - giveField(new MidFields()); - } +// public MidFields GetMid() +// { +// return Get(); +// } +// } - public MidFields GetMid() - { - return Get(); - } - } +// public class Top : Mid +// { +// protected override void YieldFields(Action giveField) +// { +// base.YieldFields(giveField); +// giveField(new TopFields()); +// } - public class Top : Mid - { - protected override void YieldFields(Action giveField) - { - base.YieldFields(giveField); - giveField(new TopFields()); - } +// public TopFields GetTop() +// { +// return Get(); +// } +// } - public TopFields GetTop() - { - return Get(); - } - } +// public class BaseFields : ITestLifecycleComponent +// { +// public string EntryPoint { get; set; } = string.Empty; +// public string Log { get; set; } = string.Empty; +// } - public class BaseFields : ITestLifecycleComponent - { - public string EntryPoint { get; set; } = string.Empty; - public string Log { get; set; } = string.Empty; - } +// public class MidFields : ITestLifecycleComponent +// { +// public string Nodes { get; set; } = string.Empty; +// } - public class MidFields : ITestLifecycleComponent - { - public string Nodes { get; set; } = string.Empty; - } - - public class TopFields : ITestLifecycleComponent - { - public string Geth { get; set; } = string.Empty; - public string Contracts { get; set; } = string.Empty; - } -} +// public class TopFields : ITestLifecycleComponent +// { +// public string Geth { get; set; } = string.Empty; +// public string Contracts { get; set; } = string.Empty; +// } +//} diff --git a/Tests/FrameworkTests/Parallelism.cs b/Tests/FrameworkTests/Parallelism.cs new file mode 100644 index 00000000..8a877f41 --- /dev/null +++ b/Tests/FrameworkTests/Parallelism.cs @@ -0,0 +1,6 @@ +using NUnit.Framework; + +[assembly: LevelOfParallelism(100)] +namespace FrameworkTests +{ +} From 373f7826bf5c87d6b53cb7b0d8478bded7ab7ff9 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 25 Apr 2025 07:21:07 +0200 Subject: [PATCH 3/7] todo: try fixture lifecycle attribute --- Tests/FrameworkTests/LifecycelyTest.cs | 329 ++++++++++++------------- 1 file changed, 164 insertions(+), 165 deletions(-) diff --git a/Tests/FrameworkTests/LifecycelyTest.cs b/Tests/FrameworkTests/LifecycelyTest.cs index 6c11e14f..e420a911 100644 --- a/Tests/FrameworkTests/LifecycelyTest.cs +++ b/Tests/FrameworkTests/LifecycelyTest.cs @@ -1,64 +1,65 @@ -//using NUnit.Framework; +using NUnit.Framework; -//namespace FrameworkTests -//{ -// [Parallelizable(ParallelScope.All)] -// [TestFixture(10)] -// [TestFixture(20)] -// [TestFixture(30)] -// public class LifecycelyTest -// { -// public LifecycelyTest(int num) -// { -// Log("ctor", GetCurrentTestName(), num); -// this.num = num; -// } +namespace FrameworkTests +{ + [Parallelizable(ParallelScope.All)] + [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] + [TestFixture(10)] + [TestFixture(20)] + [TestFixture(30)] + public class LifecycelyTest + { + public LifecycelyTest(int num) + { + Log("ctor", GetCurrentTestName(), num); + this.num = num; + } -// [SetUp] -// public void Setup() -// { -// Log(nameof(Setup), GetCurrentTestName()); -// } + [SetUp] + public void Setup() + { + Log(nameof(Setup), GetCurrentTestName()); + } -// [TearDown] -// public void TearDown() -// { -// Log(nameof(TearDown), GetCurrentTestName()); -// } + [TearDown] + public void TearDown() + { + Log(nameof(TearDown), GetCurrentTestName()); + } -// [Test] -// public void A() -// { -// Log(nameof(A), "Run"); -// SleepRandom(); -// Log(nameof(A), "Finish"); -// } + //[Test] + //public void A() + //{ + // Log(nameof(A), "Run"); + // SleepRandom(); + // Log(nameof(A), "Finish"); + //} -// [Test] -// public void B() -// { -// Log(nameof(B), "Run"); -// SleepRandom(); -// Log(nameof(B), "Finish"); -// } + //[Test] + //public void B() + //{ + // Log(nameof(B), "Run"); + // SleepRandom(); + // Log(nameof(B), "Finish"); + //} -// [Test] -// public void C() -// { -// Log(nameof(C), "Run"); -// SleepRandom(); -// Log(nameof(C), "Finish"); -// } + //[Test] + //public void C() + //{ + // Log(nameof(C), "Run"); + // SleepRandom(); + // Log(nameof(C), "Finish"); + //} -// [Test] -// [Combinatorial] -// public void Multi( -// [Values(1, 2, 3)] int num) -// { -// Log(nameof(Multi), "Run", num); -// SleepRandom(); -// Log(nameof(Multi), "Finish", num); -// } + [Test] + [Combinatorial] + public void Multi( + [Values(1, 2, 3)] int num) + { + Log(nameof(Multi), "Run", num); + SleepRandom(); + Log(nameof(Multi), "Finish", num); + } @@ -71,150 +72,148 @@ -// private static readonly Random r = new Random(); -// private readonly int num; + private static readonly Random r = new Random(); + private readonly int num; -// private void SleepRandom() -// { -// Thread.Sleep(TimeSpan.FromSeconds(5.0)); -// Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); -// } + private void SleepRandom() + { + Thread.Sleep(TimeSpan.FromSeconds(5.0)); + Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); + } -// private void Log(string scope, string msg) -// { -// ALog.Log($"{num} {scope} {msg}"); -// } + private void Log(string scope, string msg) + { + ALog.Log($"{num} {scope} {msg}"); + } -// private void Log(string scope, string msg, int num) -// { -// ALog.Log($"{this.num} {scope} {msg} {num}"); -// } + private void Log(string scope, string msg, int num) + { + ALog.Log($"{this.num} {scope} {msg} {num}"); + } -// private string GetCurrentTestName() -// { -// return $"[{TestContext.CurrentContext.Test.Name}]"; -// } -// } + private string GetCurrentTestName() + { + return $"[{TestContext.CurrentContext.Test.Name}]"; + } + } -// public class ALog -// { -// private static readonly object _lock = new object(); + public class ALog + { + private static readonly object _lock = new object(); -// public static void Log(string msg) -// { -// lock (_lock) -// { -// File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); -// } -// } -// } + public static void Log(string msg) + { + lock (_lock) + { + File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); + } + } + } + public interface ITestLifecycleComponent + { + } + public class Base + { + private readonly Dictionary> anyFields = new(); -// public class Base -// { -// private readonly Dictionary> anyFields = new(); + public void Setup() + { + var testId = 23; -// public void Setup() -// { -// var testId = 23; + var fields = new Dictionary(); + anyFields.Add(testId, fields); + YieldFields(field => + { + fields.Add(field.GetType(), field); + }); -// var fields = new Dictionary(); -// anyFields.Add(testId, fields); -// YieldFields(field => -// { -// fields.Add(field.GetType(), field); -// }); + } -// foreach (var field in fields.Values) -// { -// field.Start(); -// } -// } + public void TearDown() + { + var testId = 23; -// public void TearDown() -// { -// var testId = 23; + // foreach stop -// // foreach stop + anyFields.Remove(testId); + } -// anyFields.Remove(testId); -// } + public T Get() + { + int testId = 123; + var fields = anyFields[testId]; + var type = typeof(T); + var result = fields[type]; + return (T)result; + } -// public T Get() -// { -// int testId = 123; -// var fields = anyFields[testId]; -// var type = typeof(T); -// var result = fields[type]; -// return (T)result; -// } + public BaseFields GetBaseField() + { + return Get(); + } -// public BaseFields GetBaseField() -// { -// return Get(); -// } + protected virtual void YieldFields(Action giveField) + { + giveField(new BaseFields()); + } + } -// protected virtual void YieldFields(Action giveField) -// { -// giveField(new BaseFields()); -// } -// } + public class Mid : Base + { + protected override void YieldFields(Action giveField) + { + base.YieldFields(giveField); + giveField(new MidFields()); + } -// public class Mid : Base -// { -// protected override void YieldFields(Action giveField) -// { -// base.YieldFields(giveField); -// giveField(new MidFields()); -// } + public MidFields GetMid() + { + return Get(); + } + } -// public MidFields GetMid() -// { -// return Get(); -// } -// } + public class Top : Mid + { + protected override void YieldFields(Action giveField) + { + base.YieldFields(giveField); + giveField(new TopFields()); + } -// public class Top : Mid -// { -// protected override void YieldFields(Action giveField) -// { -// base.YieldFields(giveField); -// giveField(new TopFields()); -// } + public TopFields GetTop() + { + return Get(); + } + } -// public TopFields GetTop() -// { -// return Get(); -// } -// } + public class BaseFields : ITestLifecycleComponent + { + public string EntryPoint { get; set; } = string.Empty; + public string Log { get; set; } = string.Empty; + } -// public class BaseFields : ITestLifecycleComponent -// { -// public string EntryPoint { get; set; } = string.Empty; -// public string Log { get; set; } = string.Empty; -// } + public class MidFields : ITestLifecycleComponent + { + public string Nodes { get; set; } = string.Empty; + } -// public class MidFields : ITestLifecycleComponent -// { -// public string Nodes { get; set; } = string.Empty; -// } - -// public class TopFields : ITestLifecycleComponent -// { -// public string Geth { get; set; } = string.Empty; -// public string Contracts { get; set; } = string.Empty; -// } -//} + public class TopFields : ITestLifecycleComponent + { + public string Geth { get; set; } = string.Empty; + public string Contracts { get; set; } = string.Empty; + } +} From ad1b756db9c85f22e5a820bcb2fdc9e28661fb33 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 25 Apr 2025 11:08:38 +0200 Subject: [PATCH 4/7] Revert "while better the new plan is still a mess" This reverts commit 2dfdfac2bb52e4ff919ee5f1b3986a839d71b0a2. # Conflicts: # Tests/FrameworkTests/LifecycelyTest.cs --- .../OverwatchSupport/CodexTranscriptWriter.cs | 15 +- .../CodexTranscriptWriterConfig.cs | 4 +- .../MarketplaceAutoBootstrapDistTest.cs | 39 ++- Tests/DistTestCore/DistTest.cs | 124 +++++++--- .../DistTestLifecycleComponents.cs | 97 -------- Tests/DistTestCore/Logs/FixtureLog.cs | 13 +- Tests/DistTestCore/Logs/TestLog.cs | 2 + Tests/DistTestCore/TestLifecycle.cs | 30 +-- .../AutoBootstrapDistTest.cs | 62 ++--- Tests/ExperimentalTests/CodexDistTest.cs | 223 ++++++++++++------ .../CodexLogTrackerProvider.cs | 73 ------ Tests/FrameworkTests/Parallelism.cs | 6 - 12 files changed, 297 insertions(+), 391 deletions(-) delete mode 100644 Tests/DistTestCore/DistTestLifecycleComponents.cs delete mode 100644 Tests/ExperimentalTests/CodexLogTrackerProvider.cs delete mode 100644 Tests/FrameworkTests/Parallelism.cs diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index 53ac1496..afd8d1a2 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -23,27 +23,16 @@ namespace CodexPlugin.OverwatchSupport converter = new CodexLogConverter(writer, config, identityMap); } - public void Finalize() + public void Finalize(string outputFilepath) { log.Log("Finalizing Codex transcript..."); writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); - writer.Write(GetOutputFullPath()); + writer.Write(outputFilepath); log.Log("Done"); } - private string GetOutputFullPath() - { - var outputPath = Path.GetDirectoryName(log.GetFullName()); - if (outputPath == null) throw new Exception("Logfile path is null"); - var filename = Path.GetFileNameWithoutExtension(log.GetFullName()); - if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); - var outputFile = Path.Combine(outputPath, filename + "_" + config.OutputFilename); - if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; - return outputFile; - } - public ICodexNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs index 8c5e7bf0..247494c8 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs @@ -2,13 +2,11 @@ { public class CodexTranscriptWriterConfig { - public CodexTranscriptWriterConfig(string outputFilename, bool includeBlockReceivedEvents) + public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents) { - OutputFilename = outputFilename; IncludeBlockReceivedEvents = includeBlockReceivedEvents; } - public string OutputFilename { get; } public bool IncludeBlockReceivedEvents { get; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index f6e8a9ca..a5ec0885 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -10,41 +10,18 @@ using Utils; namespace CodexReleaseTests.MarketTests { - public class MarketplaceTestComponent : ILifecycleComponent - { - public IGethNode Geth { get; } - public ICodexContracts Contracts { get; } - - public void Start(ILifecycleComponentAccess access) - { - throw new NotImplementedException(); - } - - public void Stop(ILifecycleComponentAccess access, DistTestResult result) - { - throw new NotImplementedException(); - } - } - public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest { + private readonly Dictionary handles = new Dictionary(); protected const int StartingBalanceTST = 1000; protected const int StartingBalanceEth = 10; - protected override void CreateComponents(ILifecycleComponentCollector collector) - { - base.CreateComponents(collector); - - - collector.AddComponent(new MarketplaceTestComponent()); - } - protected override void LifecycleStart(TestLifecycle lifecycle) { base.LifecycleStart(lifecycle); var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - handles.Add(lifecycle, new MarketplaceTestComponent(geth, contracts)); + handles.Add(lifecycle, new MarketplaceHandle(geth, contracts)); } protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) @@ -346,5 +323,17 @@ namespace CodexReleaseTests.MarketTests public SlotFilledEventDTO SlotFilledEvent { get; } public ICodexNode Host { get; } } + + private class MarketplaceHandle + { + public MarketplaceHandle(IGethNode geth, ICodexContracts contracts) + { + Geth = geth; + Contracts = contracts; + } + + public IGethNode Geth { get; } + public ICodexContracts Contracts { get; } + } } } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 9a969532..534eb74a 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -19,8 +19,9 @@ namespace DistTestCore private readonly Assembly[] testAssemblies; private readonly FixtureLog fixtureLog; private readonly StatusLog statusLog; + private readonly object lifecycleLock = new object(); private readonly EntryPoint globalEntryPoint; - private readonly DistTestLifecycleComponents lifecycleComponents = new DistTestLifecycleComponents(); + private readonly Dictionary lifecycles = new Dictionary(); private readonly string deployId; public DistTest() @@ -88,12 +89,7 @@ namespace DistTestCore { try { - var testName = GetCurrentTestName(); - fixtureLog.WriteLogTag(); - Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => - { - lifecycleComponents.Setup(testName, CreateComponents); - }); + CreateNewTestLifecycle(); } catch (Exception ex) { @@ -108,7 +104,7 @@ namespace DistTestCore { try { - Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", DisposeTestLifecycle); + DisposeTestLifecycle(); } catch (Exception ex) { @@ -175,49 +171,80 @@ namespace DistTestCore { } - protected virtual void CreateComponents(ILifecycleComponentCollector collector) - { - var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); - var lifecycle = new TestLifecycle( - fixtureLog.CreateTestLog(), - configuration, - GetWebCallTimeSet(), - GetK8sTimeSet(), - testNamespace, - GetCurrentTestName(), - deployId, - ShouldWaitForCleanup()); - - collector.AddComponent(lifecycle); - } - - protected virtual void DestroyComponents(TestLifecycle lifecycle, DistTestResult testResult) + protected virtual void LifecycleStart(TestLifecycle lifecycle) { } - public T Get() where T : ILifecycleComponent + protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult) { - return lifecycleComponents.Get(GetCurrentTestName()); } - private TestLifecycle Get() + protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) { - return Get(); + } + + protected TestLifecycle Get() + { + lock (lifecycleLock) + { + return lifecycles[GetCurrentTestName()]; + } + } + + private void CreateNewTestLifecycle() + { + var testName = GetCurrentTestName(); + fixtureLog.WriteLogTag(); + Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => + { + lock (lifecycleLock) + { + var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); + var lifecycle = new TestLifecycle( + fixtureLog.CreateTestLog(), + configuration, + GetWebCallTimeSet(), + GetK8sTimeSet(), + testNamespace, + deployId, + ShouldWaitForCleanup()); + lifecycles.Add(testName, lifecycle); + LifecycleStart(lifecycle); + } + }); } private void DisposeTestLifecycle() { - var testName = GetCurrentTestName(); - var results = GetTestResult(); var lifecycle = Get(); + var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); var data = lifecycle.GetPluginMetadata(); - fixtureLog.Log($"{GetCurrentTestName()} = {results} ({testDuration})"); - statusLog.ConcludeTest(results, testDuration, data); + CollectStatusLogData(lifecycle, data); + fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); + statusLog.ConcludeTest(testResult, testDuration, data); + Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => + { + WriteEndTestLog(lifecycle.Log); - lifecycleComponents.TearDown(testName, results); + IncludeLogsOnTestFailure(lifecycle); + LifecycleStop(lifecycle, testResult); + lifecycle.DeleteAllResources(); + lifecycles.Remove(GetCurrentTestName()); + }); } + private void WriteEndTestLog(TestLog log) + { + var result = TestContext.CurrentContext.Result; + + Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}"); + if (!string.IsNullOrEmpty(result.Message)) + { + Log(result.Message); + Log($"{result.StackTrace}"); + } + } private IWebCallTimeSet GetWebCallTimeSet() { @@ -269,6 +296,28 @@ namespace DistTestCore .ToArray(); } + private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) + { + var testStatus = TestContext.CurrentContext.Result.Outcome.Status; + if (ShouldDownloadAllLogs(testStatus)) + { + lifecycle.Log.Log("Downloading all container logs..."); + lifecycle.DownloadAllLogs(); + } + } + + private bool ShouldDownloadAllLogs(TestStatus testStatus) + { + if (configuration.AlwaysDownloadContainerLogs) return true; + if (!IsDownloadingLogsEnabled()) return false; + if (testStatus == TestStatus.Failed) + { + return true; + } + + return false; + } + private string GetCurrentTestName() { return $"[{TestContext.CurrentContext.Test.Name}]"; @@ -279,8 +328,7 @@ namespace DistTestCore var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed; var status = TestContext.CurrentContext.Result.Outcome.Status.ToString(); var result = TestContext.CurrentContext.Result.Message; - var trace = TestContext.CurrentContext.Result.StackTrace; - return new DistTestResult(success, status, result ?? string.Empty, trace ?? string.Empty); + return new DistTestResult(success, status, result ?? string.Empty); } private bool IsDownloadingLogsEnabled() @@ -291,18 +339,16 @@ namespace DistTestCore public class DistTestResult { - public DistTestResult(bool success, string status, string result, string trace) + public DistTestResult(bool success, string status, string result) { Success = success; Status = status; Result = result; - Trace = trace; } public bool Success { get; } public string Status { get; } public string Result { get; } - public string Trace { get; } public override string ToString() { diff --git a/Tests/DistTestCore/DistTestLifecycleComponents.cs b/Tests/DistTestCore/DistTestLifecycleComponents.cs deleted file mode 100644 index 203a891c..00000000 --- a/Tests/DistTestCore/DistTestLifecycleComponents.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace DistTestCore -{ - public interface ILifecycleComponent - { - void Start(ILifecycleComponentAccess access); - void Stop(ILifecycleComponentAccess access, DistTestResult result); - } - - public interface ILifecycleComponentCollector - { - void AddComponent(ILifecycleComponent component); - } - - public interface ILifecycleComponentAccess - { - T Get() where T : ILifecycleComponent; - } - - public class DistTestLifecycleComponents - { - private readonly object _lock = new object(); - private readonly Dictionary> components = new(); - - public void Setup(string testName, Action initializer) - { - var newComponents = new Dictionary(); - lock (_lock) - { - components.Add(testName, newComponents); - var collector = new Collector(newComponents); - initializer(collector); - } - - var access = new ScopedAccess(this, testName); - foreach (var component in newComponents.Values) - { - component.Start(access); - } - } - - public T Get(string testName) where T : ILifecycleComponent - { - var type = typeof(T); - lock (_lock) - { - return (T)components[testName][type]; - } - } - - public void TearDown(string testName, DistTestResult result) - { - var access = new ScopedAccess(this, testName); - var closingComponents = components[testName]; - foreach (var component in closingComponents.Values) - { - component.Stop(access, result); - } - - lock (_lock) - { - components.Remove(testName); - } - } - - private class Collector : ILifecycleComponentCollector - { - private readonly Dictionary components; - - public Collector(Dictionary components) - { - this.components = components; - } - - public void AddComponent(ILifecycleComponent component) - { - components.Add(component.GetType(), component); - } - } - - private class ScopedAccess : ILifecycleComponentAccess - { - private readonly DistTestLifecycleComponents parent; - private readonly string testName; - - public ScopedAccess(DistTestLifecycleComponents parent, string testName) - { - this.parent = parent; - this.testName = testName; - } - - public T Get() where T : ILifecycleComponent - { - return parent.Get(testName); - } - } - } -} diff --git a/Tests/DistTestCore/Logs/FixtureLog.cs b/Tests/DistTestCore/Logs/FixtureLog.cs index f90147d8..9d3c77d3 100644 --- a/Tests/DistTestCore/Logs/FixtureLog.cs +++ b/Tests/DistTestCore/Logs/FixtureLog.cs @@ -4,25 +4,26 @@ namespace DistTestCore.Logs { public class FixtureLog : BaseTestLog { + private readonly ILog backingLog; + private readonly string deployId; + public FixtureLog(ILog backingLog, string deployId) : base(backingLog, deployId) { + this.backingLog = backingLog; + this.deployId = deployId; } public TestLog CreateTestLog(string name = "") { - var result = TestLog.Create(this, name); - result.Log(NameUtils.GetRawFixtureName()); - return result; + return TestLog.Create(this, name); } public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "") { var fullName = NameUtils.GetFixtureFullName(config, start, name); var log = CreateMainLog(fullName, name); - var result = new FixtureLog(log, deployId); - result.Log(NameUtils.GetRawFixtureName()); - return result; + return new FixtureLog(log, deployId); } } } diff --git a/Tests/DistTestCore/Logs/TestLog.cs b/Tests/DistTestCore/Logs/TestLog.cs index 6a334138..0dca1464 100644 --- a/Tests/DistTestCore/Logs/TestLog.cs +++ b/Tests/DistTestCore/Logs/TestLog.cs @@ -1,4 +1,5 @@ using Logging; +using System.Xml.Linq; namespace DistTestCore.Logs { @@ -7,6 +8,7 @@ namespace DistTestCore.Logs public TestLog(ILog backingLog, string methodName, string deployId, string name = "") : base(backingLog, deployId) { + backingLog.Log($"*** Begin: {methodName}"); } public static TestLog Create(FixtureLog parentLog, string name = "") diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 550b6151..3d642d20 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -10,24 +10,22 @@ using WebUtils; namespace DistTestCore { - public class TestLifecycle : IK8sHooks, ILifecycleComponent + public class TestLifecycle : IK8sHooks { private const string TestsType = "dist-tests"; private readonly EntryPoint entryPoint; private readonly Dictionary metadata; private readonly List runningContainers = new(); - private readonly string testName; private readonly string deployId; private readonly List stoppedContainerLogs = new List(); - public TestLifecycle(TestLog log, Configuration configuration, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8sTimeSet, string testNamespace, string testName, string deployId, bool waitForCleanup) + public TestLifecycle(TestLog log, Configuration configuration, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8sTimeSet, string testNamespace, string deployId, bool waitForCleanup) { Log = log; Configuration = configuration; WebCallTimeSet = webCallTimeSet; K8STimeSet = k8sTimeSet; TestNamespace = testNamespace; - this.testName = testName; TestStart = DateTime.UtcNow; entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(k8sTimeSet, this, testNamespace), configuration.GetFileManagerFolder(), webCallTimeSet, k8sTimeSet); @@ -38,28 +36,6 @@ namespace DistTestCore log.WriteLogTag(); } - public void Start(ILifecycleComponentAccess access) - { - Log.Log($"*** Begin: {testName}"); - } - - public void Stop(ILifecycleComponentAccess access, DistTestResult result) - { - Log.Log($"*** Finished: {testName} = {result.Status}"); - if (!string.IsNullOrEmpty(result.Result)) - { - Log.Log(result.Result); - Log.Log($"{result.Trace}"); - } - - if (!result.Success) - { - DownloadAllLogs(); - } - - DeleteAllResources(); - } - public DateTime TestStart { get; } public TestLog Log { get; } public Configuration Configuration { get; } @@ -69,7 +45,7 @@ namespace DistTestCore public bool WaitForCleanup { get; } public CoreInterface CoreInterface { get; } - private void DeleteAllResources() + public void DeleteAllResources() { entryPoint.Decommission( deleteKubernetesResources: true, diff --git a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs index a2477597..d8ac6805 100644 --- a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs +++ b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs @@ -4,56 +4,46 @@ using DistTestCore; namespace CodexTests { - public class AutoBootstrapComponent : ILifecycleComponent - { - public ICodexNode? BootstrapNode { get; private set; } = null; - - public void Start(ILifecycleComponentAccess access) - { - if (BootstrapNode != null) return; - - var tl = access.Get(); - var ci = tl.CoreInterface; - var testNamespace = tl.TestNamespace; - - BootstrapNode = ci.StartCodexNode(s => s.WithName("BOOTSTRAP_" + testNamespace)); - } - - public void ApplyBootstrapNode(ICodexSetup setup) - { - if (BootstrapNode == null) return; - - setup.WithBootstrapNode(BootstrapNode); - } - - public void Stop(ILifecycleComponentAccess access, DistTestResult result) - { - if (BootstrapNode == null) return; - BootstrapNode.Stop(waitTillStopped: false); - } - } - public class AutoBootstrapDistTest : CodexDistTest { + private readonly Dictionary bootstrapNodes = new Dictionary(); + private bool isBooting = false; - protected override void CreateComponents(ILifecycleComponentCollector collector) + protected override void LifecycleStart(TestLifecycle tl) { - base.CreateComponents(collector); - collector.AddComponent(new AutoBootstrapComponent()); + base.LifecycleStart(tl); + if (!bootstrapNodes.ContainsKey(tl)) + { + isBooting = true; + bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace))); + isBooting = false; + } + } + + protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) + { + bootstrapNodes.Remove(lifecycle); + base.LifecycleStop(lifecycle, result); } protected override void OnCodexSetup(ICodexSetup setup) { - Get().ApplyBootstrapNode(setup); + if (isBooting) return; + + var node = BootstrapNode; + if (node != null) setup.WithBootstrapNode(node); } protected ICodexNode BootstrapNode { get { - var bn = Get().BootstrapNode; - if (bn == null) throw new InvalidOperationException("BootstrapNode accessed before initialized."); - return bn; + var tl = Get(); + if (bootstrapNodes.TryGetValue(tl, out var node)) + { + return node; + } + throw new InvalidOperationException("Bootstrap node not yet started."); } } } diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index 40e711fc..d27216d3 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -1,5 +1,6 @@ using BlockchainUtils; using CodexClient; +using CodexClient.Hooks; using CodexContractsPlugin; using CodexNetDeployer; using CodexPlugin; @@ -16,74 +17,86 @@ using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using OverwatchTranscript; +using Utils; namespace CodexTests { - public class CodexDistTestComponents : ILifecycleComponent + public class CodexLogTrackerProvider : ICodexHooksProvider { - private readonly object nodesLock = new object(); + private readonly Action addNode; - public CodexDistTestComponents(CodexTranscriptWriter? writer) + public CodexLogTrackerProvider(Action addNode) { - Writer = writer; + this.addNode = addNode; } - public CodexTranscriptWriter? Writer { get; } - public BlockCache Cache { get; } = new(); - public List Nodes { get; } = new(); - - public void Start(ILifecycleComponentAccess access) + // See TestLifecycle.cs DownloadAllLogs() + public ICodexNodeHooks CreateHooks(string nodeName) { - var ci = access.Get().CoreInterface; - ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => + return new CodexLogTracker(addNode); + } + + public class CodexLogTracker : ICodexNodeHooks + { + private readonly Action addNode; + + public CodexLogTracker(Action addNode) { - lock (nodesLock) - { - Nodes.Add(n); - } - })); - } - - public void Stop(ILifecycleComponentAccess access, DistTestResult result) - { - var tl = access.Get(); - var log = tl.Log; - var logFiles = tl.DownloadAllLogs(); - - TeardownTranscript(log, logFiles, result); - - // todo: on not success: go to nodes and dl logs? - // or fix disttest failure log download so we can always have logs even for non-codexes? - } - - private void TeardownTranscript(TestLog log, IDownloadedLog[] logFiles, DistTestResult result) - { - if (Writer == null) return; - - Writer.AddResult(result.Success, result.Result); - - try - { - Stopwatch.Measure(log, "Transcript.ProcessLogs", () => - { - Writer.ProcessLogs(logFiles); - }); - - Stopwatch.Measure(log, $"Transcript.Finalize", () => - { - Writer.IncludeFile(log.GetFullName()); - Writer.Finalize(); - }); + this.addNode = addNode; } - catch (Exception ex) + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + addNode(node); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + } + + public void OnNodeStopping() + { + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) { - log.Error("Failure during transcript teardown: " + ex); } } } public class CodexDistTest : DistTest { + private static readonly object _lock = new object(); + private static readonly Dictionary writers = new Dictionary(); + private static readonly Dictionary blockCaches = new Dictionary(); + + // this entire structure is not good and needs to be destroyed at the earliest convenience: + private static readonly Dictionary> nodes = new Dictionary>(); + public CodexDistTest() { ProjectPlugin.Load(); @@ -99,12 +112,34 @@ namespace CodexTests localBuilder.Build(); } - protected override void CreateComponents(ILifecycleComponentCollector collector) + protected override void LifecycleStart(TestLifecycle lifecycle) { - base.CreateComponents(collector); - collector.AddComponent(new CodexDistTestComponents( - SetupTranscript() - )); + base.LifecycleStart(lifecycle); + SetupTranscript(lifecycle); + + Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => + { + lock (_lock) + { + if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List()); + nodes[lifecycle].Add(n); + } + })); + } + + protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) + { + base.LifecycleStop(lifecycle, result); + TeardownTranscript(lifecycle, result); + + if (!result.Success) + { + lock (_lock) + { + var codexNodes = nodes[lifecycle]; + foreach (var node in codexNodes) node.DownloadLog(); + } + } } public ICodexNode StartCodex() @@ -138,11 +173,6 @@ namespace CodexTests return Ci.StartGethNode(GetBlockCache(), setup); } - private BlockCache GetBlockCache() - { - return Get().Cache; - } - public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() { return new PeerConnectionTestHelpers(GetTestLog()); @@ -150,7 +180,7 @@ namespace CodexTests public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() { - return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); + return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); } public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") @@ -228,20 +258,81 @@ namespace CodexTests return null; } - private CodexTranscriptWriter? SetupTranscript() + private void SetupTranscript(TestLifecycle lifecycle) { var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return null; + if (attr == null) return; var config = new CodexTranscriptWriterConfig( - attr.OutputFilename, attr.IncludeBlockReceivedEvents ); - var log = new LogPrefixer(GetTestLog(), "(Transcript) "); + var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); Ci.AddCodexHooksProvider(writer); - return writer; + lock (_lock) + { + writers.Add(lifecycle, writer); + } + } + + private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result) + { + var attr = GetTranscriptAttributeOfCurrentTest(); + if (attr == null) return; + + var outputFilepath = GetOutputFullPath(lifecycle, attr); + + CodexTranscriptWriter writer = null!; + lock (_lock) + { + writer = writers[lifecycle]; + writers.Remove(lifecycle); + } + + writer.AddResult(result.Success, result.Result); + + try + { + Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () => + { + writer.ProcessLogs(lifecycle.DownloadAllLogs()); + }); + + Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () => + { + writer.IncludeFile(lifecycle.Log.GetFullName()); + writer.Finalize(outputFilepath); + }); + } + catch (Exception ex) + { + lifecycle.Log.Error("Failure during transcript teardown: " + ex); + } + } + + private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr) + { + var outputPath = Path.GetDirectoryName(lifecycle.Log.GetFullName()); + if (outputPath == null) throw new Exception("Logfile path is null"); + var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.GetFullName()); + if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); + var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename); + if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; + return outputFile; + } + + private BlockCache GetBlockCache() + { + var lifecycle = Get(); + lock (_lock) + { + if (!blockCaches.ContainsKey(lifecycle)) + { + blockCaches[lifecycle] = new BlockCache(); + } + } + return blockCaches[lifecycle]; } } diff --git a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs deleted file mode 100644 index 1b2fdd66..00000000 --- a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs +++ /dev/null @@ -1,73 +0,0 @@ -using CodexClient; -using CodexClient.Hooks; -using Utils; - -namespace CodexTests -{ - public class CodexLogTrackerProvider : ICodexHooksProvider - { - private readonly Action addNode; - - public CodexLogTrackerProvider(Action addNode) - { - this.addNode = addNode; - } - - // See TestLifecycle.cs DownloadAllLogs() - public ICodexNodeHooks CreateHooks(string nodeName) - { - return new CodexLogTracker(addNode); - } - - public class CodexLogTracker : ICodexNodeHooks - { - private readonly Action addNode; - - public CodexLogTracker(Action addNode) - { - this.addNode = addNode; - } - - public void OnFileDownloaded(ByteSize size, ContentId cid) - { - } - - public void OnFileDownloading(ContentId cid) - { - } - - public void OnFileUploaded(string uid, ByteSize size, ContentId cid) - { - } - - public void OnFileUploading(string uid, ByteSize size) - { - } - - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) - { - addNode(node); - } - - public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) - { - } - - public void OnNodeStopping() - { - } - - public void OnStorageAvailabilityCreated(StorageAvailability response) - { - } - - public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) - { - } - - public void OnStorageContractUpdated(StoragePurchase purchaseStatus) - { - } - } - } -} diff --git a/Tests/FrameworkTests/Parallelism.cs b/Tests/FrameworkTests/Parallelism.cs deleted file mode 100644 index 8a877f41..00000000 --- a/Tests/FrameworkTests/Parallelism.cs +++ /dev/null @@ -1,6 +0,0 @@ -using NUnit.Framework; - -[assembly: LevelOfParallelism(100)] -namespace FrameworkTests -{ -} From dd888f30e363562a9db1d2d90e497f3343f73b34 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 25 Apr 2025 15:42:13 +0200 Subject: [PATCH 5/7] excellent cleanup --- .../OverwatchSupport/CodexTranscriptWriter.cs | 15 +- .../CodexTranscriptWriterConfig.cs | 4 +- .../MarketplaceAutoBootstrapDistTest.cs | 20 +- .../NodeTests/BasicInfoTests.cs | 8 +- Tests/CodexReleaseTests/Parallelism.cs | 2 +- Tests/DistTestCore/DistTest.cs | 154 +++++--------- Tests/DistTestCore/Global.cs | 60 ++++++ Tests/DistTestCore/Logs/FixtureLog.cs | 9 +- Tests/DistTestCore/Logs/TestLog.cs | 5 +- Tests/DistTestCore/NameUtils.cs | 14 +- .../AutoBootstrapDistTest.cs | 37 +--- Tests/ExperimentalTests/CodexDistTest.cs | 190 ++++-------------- .../CodexLogTrackerProvider.cs | 73 +++++++ 13 files changed, 258 insertions(+), 333 deletions(-) create mode 100644 Tests/DistTestCore/Global.cs create mode 100644 Tests/ExperimentalTests/CodexLogTrackerProvider.cs diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index afd8d1a2..95b8dde6 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -23,16 +23,27 @@ namespace CodexPlugin.OverwatchSupport converter = new CodexLogConverter(writer, config, identityMap); } - public void Finalize(string outputFilepath) + public void FinalizeWriter() { log.Log("Finalizing Codex transcript..."); writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); - writer.Write(outputFilepath); + writer.Write(GetOutputFullPath()); log.Log("Done"); } + private string GetOutputFullPath() + { + var outputPath = Path.GetDirectoryName(log.GetFullName()); + if (outputPath == null) throw new Exception("Logfile path is null"); + var filename = Path.GetFileNameWithoutExtension(log.GetFullName()); + if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); + var outputFile = Path.Combine(outputPath, filename + "_" + config.OutputPath); + if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; + return outputFile; + } + public ICodexNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs index 247494c8..112f5b16 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs @@ -2,11 +2,13 @@ { public class CodexTranscriptWriterConfig { - public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents) + public CodexTranscriptWriterConfig(string outputPath, bool includeBlockReceivedEvents) { + OutputPath = outputPath; IncludeBlockReceivedEvents = includeBlockReceivedEvents; } + public string OutputPath { get; } public bool IncludeBlockReceivedEvents { get; } } } diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index a5ec0885..96f84813 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -3,41 +3,35 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; -using DistTestCore; using GethPlugin; using Nethereum.Hex.HexConvertors.Extensions; +using NUnit.Framework; using Utils; namespace CodexReleaseTests.MarketTests { public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest { - private readonly Dictionary handles = new Dictionary(); + private MarketplaceHandle handle = null!; protected const int StartingBalanceTST = 1000; protected const int StartingBalanceEth = 10; - protected override void LifecycleStart(TestLifecycle lifecycle) + [SetUp] + public void SetupMarketplace() { - base.LifecycleStart(lifecycle); var geth = StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); - handles.Add(lifecycle, new MarketplaceHandle(geth, contracts)); - } - - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) - { - handles.Remove(lifecycle); - base.LifecycleStop(lifecycle, result); + handle = new MarketplaceHandle(geth, contracts); } protected IGethNode GetGeth() { - return handles[Get()].Geth; + return handle.Geth; } protected ICodexContracts GetContracts() { - return handles[Get()].Contracts; + return handle.Contracts; } protected TimeSpan GetPeriodDuration() diff --git a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs b/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs index 9b9f4bbf..f6cba228 100644 --- a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs +++ b/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs @@ -1,11 +1,5 @@ -using CodexPlugin; -using CodexTests; +using CodexTests; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Utils; namespace CodexReleaseTests.NodeTests diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs index f6e68cc0..d589af32 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/CodexReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -[assembly: LevelOfParallelism(1)] +[assembly: LevelOfParallelism(10)] namespace CodexReleaseTests { } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 534eb74a..93732405 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -12,70 +12,45 @@ using Assert = NUnit.Framework.Assert; namespace DistTestCore { [Parallelizable(ParallelScope.All)] + [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] public abstract class DistTest { - private const string TestNamespacePrefix = "cdx-"; - private readonly Configuration configuration = new Configuration(); - private readonly Assembly[] testAssemblies; + private static readonly Global global = new Global(); private readonly FixtureLog fixtureLog; private readonly StatusLog statusLog; - private readonly object lifecycleLock = new object(); - private readonly EntryPoint globalEntryPoint; - private readonly Dictionary lifecycles = new Dictionary(); - private readonly string deployId; - + private readonly TestLifecycle lifecycle; + private readonly string deployId = NameUtils.MakeDeployId(); + public DistTest() { - var assemblies = AppDomain.CurrentDomain.GetAssemblies(); - testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); - - deployId = NameUtils.MakeDeployId(); - - var logConfig = configuration.GetLogConfig(); + var logConfig = global.Configuration.GetLogConfig(); var startTime = DateTime.UtcNow; fixtureLog = FixtureLog.Create(logConfig, startTime, deployId); statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId); - globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultK8sTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder()); + fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus()); + + lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(startTime), global.Configuration, + GetWebCallTimeSet(), + GetK8sTimeSet(), + Global.TestNamespacePrefix + Guid.NewGuid().ToString(), + deployId, + ShouldWaitForCleanup() + ); Initialize(fixtureLog); } [OneTimeSetUp] - public void GlobalSetup() + public static void GlobalSetup() { - fixtureLog.Log($"Starting..."); - globalEntryPoint.Announce(); - - // Previous test run may have been interrupted. - // Begin by cleaning everything up. - try - { - Stopwatch.Measure(fixtureLog, "Global setup", () => - { - globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true); - }); - } - catch (Exception ex) - { - GlobalTestFailure.HasFailed = true; - fixtureLog.Error($"Global setup cleanup failed with: {ex}"); - throw; - } - - fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus()); - fixtureLog.Log("Global setup cleanup successful"); + global.Setup(); } [OneTimeTearDown] - public void GlobalTearDown() + public static void GlobalTearDown() { - globalEntryPoint.Decommission( - // There shouldn't be any of either, but clean everything up regardless. - deleteKubernetesResources: true, - deleteTrackedFiles: true, - waitTillDone: true - ); + global.TearDown(); } [SetUp] @@ -85,18 +60,6 @@ namespace DistTestCore { Assert.Inconclusive("Skip test: Previous test failed during clean up."); } - else - { - try - { - CreateNewTestLifecycle(); - } - catch (Exception ex) - { - fixtureLog.Error("Setup failed: " + ex); - GlobalTestFailure.HasFailed = true; - } - } } [TearDown] @@ -117,18 +80,18 @@ namespace DistTestCore { get { - return Get().CoreInterface; + return lifecycle.CoreInterface; } } public TrackedFile GenerateTestFile(ByteSize size, string label = "") { - return Get().GenerateTestFile(size, label); + return lifecycle.GenerateTestFile(size, label); } public TrackedFile GenerateTestFile(Action options, string label = "") { - return Get().GenerateTestFile(options, label); + return lifecycle.GenerateTestFile(options, label); } /// @@ -137,12 +100,22 @@ namespace DistTestCore /// public void ScopedTestFiles(Action action) { - Get().GetFileManager().ScopedFiles(action); + lifecycle.GetFileManager().ScopedFiles(action); } public ILog GetTestLog() { - return Get().Log; + return lifecycle.Log; + } + + public IFileManager GetFileManager() + { + return lifecycle.GetFileManager(); + } + + public string GetTestNamespace() + { + return lifecycle.TestNamespace; } public void Log(string msg) @@ -159,64 +132,24 @@ namespace DistTestCore public void Measure(string name, Action action) { - Stopwatch.Measure(Get().Log, name, action); + Stopwatch.Measure(lifecycle.Log, name, action); } protected TimeRange GetTestRunTimeRange() { - return new TimeRange(Get().TestStart, DateTime.UtcNow); + return new TimeRange(lifecycle.TestStart, DateTime.UtcNow); } protected virtual void Initialize(FixtureLog fixtureLog) { } - protected virtual void LifecycleStart(TestLifecycle lifecycle) - { - } - - protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult) - { - } - protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) { } - protected TestLifecycle Get() - { - lock (lifecycleLock) - { - return lifecycles[GetCurrentTestName()]; - } - } - - private void CreateNewTestLifecycle() - { - var testName = GetCurrentTestName(); - fixtureLog.WriteLogTag(); - Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => - { - lock (lifecycleLock) - { - var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); - var lifecycle = new TestLifecycle( - fixtureLog.CreateTestLog(), - configuration, - GetWebCallTimeSet(), - GetK8sTimeSet(), - testNamespace, - deployId, - ShouldWaitForCleanup()); - lifecycles.Add(testName, lifecycle); - LifecycleStart(lifecycle); - } - }); - } - private void DisposeTestLifecycle() { - var lifecycle = Get(); var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); var data = lifecycle.GetPluginMetadata(); @@ -228,9 +161,7 @@ namespace DistTestCore WriteEndTestLog(lifecycle.Log); IncludeLogsOnTestFailure(lifecycle); - LifecycleStop(lifecycle, testResult); lifecycle.DeleteAllResources(); - lifecycles.Remove(GetCurrentTestName()); }); } @@ -287,7 +218,7 @@ namespace DistTestCore var className = currentTest.ClassName; var methodName = currentTest.MethodName; - var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); + var testClasses = global.TestAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); return testMethods.Select(m => m.GetCustomAttribute()) @@ -296,19 +227,24 @@ namespace DistTestCore .ToArray(); } + protected IDownloadedLog[] DownloadAllLogs() + { + return lifecycle.DownloadAllLogs(); + } + private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) { var testStatus = TestContext.CurrentContext.Result.Outcome.Status; if (ShouldDownloadAllLogs(testStatus)) { lifecycle.Log.Log("Downloading all container logs..."); - lifecycle.DownloadAllLogs(); + DownloadAllLogs(); } } private bool ShouldDownloadAllLogs(TestStatus testStatus) { - if (configuration.AlwaysDownloadContainerLogs) return true; + if (global.Configuration.AlwaysDownloadContainerLogs) return true; if (!IsDownloadingLogsEnabled()) return false; if (testStatus == TestStatus.Failed) { @@ -323,7 +259,7 @@ namespace DistTestCore return $"[{TestContext.CurrentContext.Test.Name}]"; } - private DistTestResult GetTestResult() + public DistTestResult GetTestResult() { var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed; var status = TestContext.CurrentContext.Result.Outcome.Status.ToString(); diff --git a/Tests/DistTestCore/Global.cs b/Tests/DistTestCore/Global.cs new file mode 100644 index 00000000..4ba1374b --- /dev/null +++ b/Tests/DistTestCore/Global.cs @@ -0,0 +1,60 @@ +using System.Reflection; +using Core; +using Logging; + +namespace DistTestCore +{ + public class Global + { + public const string TestNamespacePrefix = "cdx-"; + public Configuration Configuration { get; } = new Configuration(); + + public Assembly[] TestAssemblies { get; } + private readonly EntryPoint globalEntryPoint; + private readonly ILog log; + + public Global() + { + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + TestAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); + + log = new ConsoleLog(); + globalEntryPoint = new EntryPoint( + log, + Configuration.GetK8sConfiguration( + new DefaultK8sTimeSet(), + TestNamespacePrefix + ), + Configuration.GetFileManagerFolder() + ); + } + + public void Setup() + { + try + { + Stopwatch.Measure(log, "Global setup", () => + { + globalEntryPoint.Announce(); + globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true); + }); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + log.Error($"Global setup cleanup failed with: {ex}"); + throw; + } + } + + public void TearDown() + { + globalEntryPoint.Decommission( + // There shouldn't be any of either, but clean everything up regardless. + deleteKubernetesResources: true, + deleteTrackedFiles: true, + waitTillDone: true + ); + } + } +} diff --git a/Tests/DistTestCore/Logs/FixtureLog.cs b/Tests/DistTestCore/Logs/FixtureLog.cs index 9d3c77d3..5978a602 100644 --- a/Tests/DistTestCore/Logs/FixtureLog.cs +++ b/Tests/DistTestCore/Logs/FixtureLog.cs @@ -4,19 +4,14 @@ namespace DistTestCore.Logs { public class FixtureLog : BaseTestLog { - private readonly ILog backingLog; - private readonly string deployId; - public FixtureLog(ILog backingLog, string deployId) : base(backingLog, deployId) { - this.backingLog = backingLog; - this.deployId = deployId; } - public TestLog CreateTestLog(string name = "") + public TestLog CreateTestLog(DateTime start, string name = "") { - return TestLog.Create(this, name); + return TestLog.Create(this, start, name); } public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "") diff --git a/Tests/DistTestCore/Logs/TestLog.cs b/Tests/DistTestCore/Logs/TestLog.cs index 0dca1464..5c38f951 100644 --- a/Tests/DistTestCore/Logs/TestLog.cs +++ b/Tests/DistTestCore/Logs/TestLog.cs @@ -1,5 +1,4 @@ using Logging; -using System.Xml.Linq; namespace DistTestCore.Logs { @@ -11,9 +10,9 @@ namespace DistTestCore.Logs backingLog.Log($"*** Begin: {methodName}"); } - public static TestLog Create(FixtureLog parentLog, string name = "") + public static TestLog Create(FixtureLog parentLog, DateTime start, string name = "") { - var methodName = NameUtils.GetTestMethodName(name); + var methodName = NameUtils.GetTestLogFileName(start, name); var fullName = Path.Combine(parentLog.GetFullName(), methodName); var backingLog = CreateMainLog(fullName, name); return new TestLog(backingLog, methodName, parentLog.DeployId); diff --git a/Tests/DistTestCore/NameUtils.cs b/Tests/DistTestCore/NameUtils.cs index 55489db9..44919d51 100644 --- a/Tests/DistTestCore/NameUtils.cs +++ b/Tests/DistTestCore/NameUtils.cs @@ -5,6 +5,11 @@ namespace DistTestCore { public static class NameUtils { + public static string GetTestLogFileName(DateTime start, string name = "") + { + return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{GetTestMethodName(name)}"; + } + public static string GetTestMethodName(string name = "") { if (!string.IsNullOrEmpty(name)) return name; @@ -16,7 +21,7 @@ namespace DistTestCore public static string GetFixtureFullName(LogConfig config, DateTime start, string name) { var folder = DetermineFolder(config, start); - var fixtureName = GetFixtureName(name, start); + var fixtureName = GetRawFixtureName(); return Path.Combine(folder, fixtureName); } @@ -85,13 +90,6 @@ namespace DistTestCore Pad(start.Day)); } - private static string GetFixtureName(string name, DateTime start) - { - var fixtureName = GetRawFixtureName(); - if (!string.IsNullOrEmpty(name)) fixtureName = name; - return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{fixtureName.Replace('.', '-')}"; - } - private static string Pad(int n) { return n.ToString().PadLeft(2, '0'); diff --git a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs index d8ac6805..93fcf6f7 100644 --- a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs +++ b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs @@ -1,29 +1,27 @@ using CodexClient; using CodexPlugin; -using DistTestCore; +using NUnit.Framework; namespace CodexTests { public class AutoBootstrapDistTest : CodexDistTest { - private readonly Dictionary bootstrapNodes = new Dictionary(); private bool isBooting = false; - protected override void LifecycleStart(TestLifecycle tl) + public ICodexNode BootstrapNode { get; private set; } = null!; + + [SetUp] + public void SetupBootstrapNode() { - base.LifecycleStart(tl); - if (!bootstrapNodes.ContainsKey(tl)) - { - isBooting = true; - bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace))); - isBooting = false; - } + isBooting = true; + BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP_" + GetTestNamespace())); + isBooting = false; } - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) + [TearDown] + public void TearDownBootstrapNode() { - bootstrapNodes.Remove(lifecycle); - base.LifecycleStop(lifecycle, result); + BootstrapNode.Stop(waitTillStopped: false); } protected override void OnCodexSetup(ICodexSetup setup) @@ -33,18 +31,5 @@ namespace CodexTests var node = BootstrapNode; if (node != null) setup.WithBootstrapNode(node); } - - protected ICodexNode BootstrapNode - { - get - { - var tl = Get(); - if (bootstrapNodes.TryGetValue(tl, out var node)) - { - return node; - } - throw new InvalidOperationException("Bootstrap node not yet started."); - } - } } } diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index d27216d3..5ecae330 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -1,6 +1,5 @@ using BlockchainUtils; using CodexClient; -using CodexClient.Hooks; using CodexContractsPlugin; using CodexNetDeployer; using CodexPlugin; @@ -17,85 +16,14 @@ using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using OverwatchTranscript; -using Utils; namespace CodexTests { - public class CodexLogTrackerProvider : ICodexHooksProvider - { - private readonly Action addNode; - - public CodexLogTrackerProvider(Action addNode) - { - this.addNode = addNode; - } - - // See TestLifecycle.cs DownloadAllLogs() - public ICodexNodeHooks CreateHooks(string nodeName) - { - return new CodexLogTracker(addNode); - } - - public class CodexLogTracker : ICodexNodeHooks - { - private readonly Action addNode; - - public CodexLogTracker(Action addNode) - { - this.addNode = addNode; - } - - public void OnFileDownloaded(ByteSize size, ContentId cid) - { - } - - public void OnFileDownloading(ContentId cid) - { - } - - public void OnFileUploaded(string uid, ByteSize size, ContentId cid) - { - } - - public void OnFileUploading(string uid, ByteSize size) - { - } - - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) - { - addNode(node); - } - - public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) - { - } - - public void OnNodeStopping() - { - } - - public void OnStorageAvailabilityCreated(StorageAvailability response) - { - } - - public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) - { - } - - public void OnStorageContractUpdated(StoragePurchase purchaseStatus) - { - } - } - } - public class CodexDistTest : DistTest { - private static readonly object _lock = new object(); - private static readonly Dictionary writers = new Dictionary(); - private static readonly Dictionary blockCaches = new Dictionary(); - - // this entire structure is not good and needs to be destroyed at the earliest convenience: - private static readonly Dictionary> nodes = new Dictionary>(); + private readonly BlockCache blockCache = new BlockCache(); + private readonly List nodes = new List(); + private CodexTranscriptWriter? writer; public CodexDistTest() { @@ -105,41 +33,26 @@ namespace CodexTests ProjectPlugin.Load(); } + [SetUp] + public void SetupCodexDistTest() + { + writer = SetupTranscript(); + + } + + [TearDown] + public void TearDownCodexDistTest() + { + TeardownTranscript(); + } + protected override void Initialize(FixtureLog fixtureLog) { var localBuilder = new LocalCodexBuilder(fixtureLog); localBuilder.Intialize(); localBuilder.Build(); - } - protected override void LifecycleStart(TestLifecycle lifecycle) - { - base.LifecycleStart(lifecycle); - SetupTranscript(lifecycle); - - Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => - { - lock (_lock) - { - if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List()); - nodes[lifecycle].Add(n); - } - })); - } - - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) - { - base.LifecycleStop(lifecycle, result); - TeardownTranscript(lifecycle, result); - - if (!result.Success) - { - lock (_lock) - { - var codexNodes = nodes[lifecycle]; - foreach (var node in codexNodes) node.DownloadLog(); - } - } + Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(nodes.Add)); } public ICodexNode StartCodex() @@ -170,7 +83,7 @@ namespace CodexTests public IGethNode StartGethNode(Action setup) { - return Ci.StartGethNode(GetBlockCache(), setup); + return Ci.StartGethNode(blockCache, setup); } public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() @@ -180,7 +93,7 @@ namespace CodexTests public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() { - return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); + return new PeerDownloadTestHelpers(GetTestLog(), GetFileManager()); } public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") @@ -258,82 +171,47 @@ namespace CodexTests return null; } - private void SetupTranscript(TestLifecycle lifecycle) + private CodexTranscriptWriter? SetupTranscript() { var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; + if (attr == null) return null; var config = new CodexTranscriptWriterConfig( + attr.OutputFilename, attr.IncludeBlockReceivedEvents ); - var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); + var log = new LogPrefixer(GetTestLog(), "(Transcript) "); var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); Ci.AddCodexHooksProvider(writer); - lock (_lock) - { - writers.Add(lifecycle, writer); - } + return writer; } - private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result) + private void TeardownTranscript() { - var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; - - var outputFilepath = GetOutputFullPath(lifecycle, attr); - - CodexTranscriptWriter writer = null!; - lock (_lock) - { - writer = writers[lifecycle]; - writers.Remove(lifecycle); - } + if (writer == null) return; + var result = GetTestResult(); + var log = GetTestLog(); writer.AddResult(result.Success, result.Result); - try { - Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () => + Stopwatch.Measure(log, "Transcript.ProcessLogs", () => { - writer.ProcessLogs(lifecycle.DownloadAllLogs()); + writer.ProcessLogs(DownloadAllLogs()); }); - Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () => + Stopwatch.Measure(log, $"Transcript.FinalizeWriter", () => { - writer.IncludeFile(lifecycle.Log.GetFullName()); - writer.Finalize(outputFilepath); + writer.IncludeFile(log.GetFullName()); + writer.FinalizeWriter(); }); } catch (Exception ex) { - lifecycle.Log.Error("Failure during transcript teardown: " + ex); + log.Error("Failure during transcript teardown: " + ex); } } - - private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr) - { - var outputPath = Path.GetDirectoryName(lifecycle.Log.GetFullName()); - if (outputPath == null) throw new Exception("Logfile path is null"); - var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.GetFullName()); - if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); - var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename); - if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; - return outputFile; - } - - private BlockCache GetBlockCache() - { - var lifecycle = Get(); - lock (_lock) - { - if (!blockCaches.ContainsKey(lifecycle)) - { - blockCaches[lifecycle] = new BlockCache(); - } - } - return blockCaches[lifecycle]; - } } [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] diff --git a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs new file mode 100644 index 00000000..1b2fdd66 --- /dev/null +++ b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs @@ -0,0 +1,73 @@ +using CodexClient; +using CodexClient.Hooks; +using Utils; + +namespace CodexTests +{ + public class CodexLogTrackerProvider : ICodexHooksProvider + { + private readonly Action addNode; + + public CodexLogTrackerProvider(Action addNode) + { + this.addNode = addNode; + } + + // See TestLifecycle.cs DownloadAllLogs() + public ICodexNodeHooks CreateHooks(string nodeName) + { + return new CodexLogTracker(addNode); + } + + public class CodexLogTracker : ICodexNodeHooks + { + private readonly Action addNode; + + public CodexLogTracker(Action addNode) + { + this.addNode = addNode; + } + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + addNode(node); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + } + + public void OnNodeStopping() + { + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + { + } + } + } +} From 24a25292b8f12ef2788167a628499c959a3906e5 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 25 Apr 2025 16:13:01 +0200 Subject: [PATCH 6/7] fixes transcript writing --- Framework/Utils/Str.cs | 5 ++++- Tests/ExperimentalTests/CodexDistTest.cs | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Framework/Utils/Str.cs b/Framework/Utils/Str.cs index aea19ec8..ab8fe46d 100644 --- a/Framework/Utils/Str.cs +++ b/Framework/Utils/Str.cs @@ -4,8 +4,11 @@ { public static string Between(string input, string open, string close) { - var openIndex = input.IndexOf(open) + open.Length; + var openI = input.IndexOf(open); + if (openI == -1) return input; + var openIndex = openI + open.Length; var closeIndex = input.LastIndexOf(close); + if (closeIndex == -1) return input; return input.Substring(openIndex, closeIndex - openIndex); } diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index 5ecae330..422b5bd7 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -37,7 +37,6 @@ namespace CodexTests public void SetupCodexDistTest() { writer = SetupTranscript(); - } [TearDown] @@ -203,7 +202,7 @@ namespace CodexTests Stopwatch.Measure(log, $"Transcript.FinalizeWriter", () => { - writer.IncludeFile(log.GetFullName()); + writer.IncludeFile(log.GetFullName() + ".log"); writer.FinalizeWriter(); }); } From acf5436d388448886d201fab4d296a1329fe72cd Mon Sep 17 00:00:00 2001 From: ThatBen Date: Tue, 29 Apr 2025 12:05:33 +0200 Subject: [PATCH 7/7] remove lifecycle test test --- Tests/FrameworkTests/LifecycelyTest.cs | 219 ------------------------- 1 file changed, 219 deletions(-) delete mode 100644 Tests/FrameworkTests/LifecycelyTest.cs diff --git a/Tests/FrameworkTests/LifecycelyTest.cs b/Tests/FrameworkTests/LifecycelyTest.cs deleted file mode 100644 index e420a911..00000000 --- a/Tests/FrameworkTests/LifecycelyTest.cs +++ /dev/null @@ -1,219 +0,0 @@ -using NUnit.Framework; - -namespace FrameworkTests -{ - [Parallelizable(ParallelScope.All)] - [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] - [TestFixture(10)] - [TestFixture(20)] - [TestFixture(30)] - public class LifecycelyTest - { - public LifecycelyTest(int num) - { - Log("ctor", GetCurrentTestName(), num); - this.num = num; - } - - [SetUp] - public void Setup() - { - Log(nameof(Setup), GetCurrentTestName()); - } - - [TearDown] - public void TearDown() - { - Log(nameof(TearDown), GetCurrentTestName()); - } - - //[Test] - //public void A() - //{ - // Log(nameof(A), "Run"); - // SleepRandom(); - // Log(nameof(A), "Finish"); - //} - - //[Test] - //public void B() - //{ - // Log(nameof(B), "Run"); - // SleepRandom(); - // Log(nameof(B), "Finish"); - //} - - //[Test] - //public void C() - //{ - // Log(nameof(C), "Run"); - // SleepRandom(); - // Log(nameof(C), "Finish"); - //} - - [Test] - [Combinatorial] - public void Multi( - [Values(1, 2, 3)] int num) - { - Log(nameof(Multi), "Run", num); - SleepRandom(); - Log(nameof(Multi), "Finish", num); - } - - - - - - - - - - - - - private static readonly Random r = new Random(); - private readonly int num; - - private void SleepRandom() - { - Thread.Sleep(TimeSpan.FromSeconds(5.0)); - Thread.Sleep(TimeSpan.FromMilliseconds(r.Next(100, 1000))); - } - - private void Log(string scope, string msg) - { - ALog.Log($"{num} {scope} {msg}"); - } - - private void Log(string scope, string msg, int num) - { - ALog.Log($"{this.num} {scope} {msg} {num}"); - } - - private string GetCurrentTestName() - { - return $"[{TestContext.CurrentContext.Test.Name}]"; - } - } - - - - - public class ALog - { - private static readonly object _lock = new object(); - - public static void Log(string msg) - { - lock (_lock) - { - File.AppendAllLines("C:\\Users\\vexor\\Desktop\\Alog.txt", [msg]); - } - } - } - - - - - - - public interface ITestLifecycleComponent - { - } - - - - - - public class Base - { - private readonly Dictionary> anyFields = new(); - - public void Setup() - { - var testId = 23; - - var fields = new Dictionary(); - anyFields.Add(testId, fields); - YieldFields(field => - { - fields.Add(field.GetType(), field); - }); - - } - - public void TearDown() - { - var testId = 23; - - // foreach stop - - anyFields.Remove(testId); - } - - public T Get() - { - int testId = 123; - var fields = anyFields[testId]; - var type = typeof(T); - var result = fields[type]; - return (T)result; - } - - public BaseFields GetBaseField() - { - return Get(); - } - - protected virtual void YieldFields(Action giveField) - { - giveField(new BaseFields()); - } - } - - public class Mid : Base - { - protected override void YieldFields(Action giveField) - { - base.YieldFields(giveField); - giveField(new MidFields()); - } - - public MidFields GetMid() - { - return Get(); - } - } - - public class Top : Mid - { - protected override void YieldFields(Action giveField) - { - base.YieldFields(giveField); - giveField(new TopFields()); - } - - public TopFields GetTop() - { - return Get(); - } - } - - public class BaseFields : ITestLifecycleComponent - { - public string EntryPoint { get; set; } = string.Empty; - public string Log { get; set; } = string.Empty; - } - - public class MidFields : ITestLifecycleComponent - { - public string Nodes { get; set; } = string.Empty; - } - - public class TopFields : ITestLifecycleComponent - { - public string Geth { get; set; } = string.Empty; - public string Contracts { get; set; } = string.Empty; - } -}