diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 0e699e1..5be4ff8 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -15,7 +15,7 @@ namespace ContinuousTests public ContinuousTestRunner(string[] args, CancellationToken cancelToken) { config = configLoader.Load(args); - startupChecker = new StartupChecker(config); + startupChecker = new StartupChecker(config, cancelToken); this.cancelToken = cancelToken; } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 9316789..f8e6583 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -62,7 +62,6 @@ namespace ContinuousTests { try { - OverviewLog(" > Starting test. " + FuturesInfo()); RunTestMoments(); if (!config.KeepPassedTestLogs) fixtureLog.Delete(); diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 4c33ccd..8e9357a 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -1,7 +1,6 @@ using DistTestCore.Codex; using DistTestCore; using Logging; -using NUnit.Framework.Internal; namespace ContinuousTests { @@ -10,10 +9,12 @@ namespace ContinuousTests private readonly TestFactory testFactory = new TestFactory(); private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory(); private readonly Configuration config; + private readonly CancellationToken cancelToken; - public StartupChecker(Configuration config) + public StartupChecker(Configuration config, CancellationToken cancelToken) { this.config = config; + this.cancelToken = cancelToken; } public void Check() @@ -36,6 +37,8 @@ namespace ContinuousTests } foreach (var test in tests) { + cancelToken.ThrowIfCancellationRequested(); + var handle = new TestHandle(test); handle.GetEarliestMoment(); handle.GetLastMoment(); @@ -59,6 +62,8 @@ namespace ContinuousTests var pass = true; foreach (var n in nodes) { + cancelToken.ThrowIfCancellationRequested(); + log.Log($"Checking '{n.Address.Host}'..."); if (EnsureOnline(n)) diff --git a/ContinuousTests/Tests/ThresholdChecks.cs b/ContinuousTests/Tests/ThresholdChecks.cs new file mode 100644 index 0000000..f1b8167 --- /dev/null +++ b/ContinuousTests/Tests/ThresholdChecks.cs @@ -0,0 +1,43 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class ThresholdChecks : ContinuousTest + { + public override int RequiredNumberOfNodes => 1; + public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; + + [TestMoment(t: 0)] + public void CheckAllThresholds() + { + var allNodes = CreateAccessToAllNodes(); + foreach (var n in allNodes) CheckThresholds(n); + } + + private void CheckThresholds(CodexAccess n) + { + var breaches = n.GetDebugThresholdBreaches(); + if (breaches.breaches.Any()) + { + Assert.Fail(string.Join(",", breaches.breaches.Select(b => FormatBreach(n, b)))); + } + } + + private string FormatBreach(CodexAccess n, string breach) + { + return $"{n.Container.Name} = '{breach}'"; + } + + private CodexAccess[] CreateAccessToAllNodes() + { + // Normally, a continuous test accesses only a subset of the nodes in the deployment. + // This time, we want to check all of them. + var factory = new CodexAccessFactory(); + var allContainers = Configuration.CodexDeployment.CodexContainers; + return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet()); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 0ec046c..fde9997 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -53,6 +53,11 @@ namespace DistTestCore.Codex return Http().HttpGetJson("debug/futures").futures; } + public CodexDebugThresholdBreaches GetDebugThresholdBreaches() + { + return Http().HttpGetJson("debug/loop"); + } + public string UploadFile(FileStream fileStream) { return Http().HttpPostStream("upload", fileStream); diff --git a/DistTestCore/Codex/CodexApiTypes.cs b/DistTestCore/Codex/CodexApiTypes.cs index 835df10..cde05fd 100644 --- a/DistTestCore/Codex/CodexApiTypes.cs +++ b/DistTestCore/Codex/CodexApiTypes.cs @@ -76,6 +76,11 @@ namespace DistTestCore.Codex public string address { get; set; } = string.Empty; } + public class CodexDebugThresholdBreaches + { + public string[] breaches { get; set; } = Array.Empty(); + } + public class CodexSalesAvailabilityRequest { public string size { get; set; } = string.Empty;