Adds threshold checking test

ContinuousTests/ContinuousTestRunner.cs

@@ -15,7 +15,7 @@ namespace ContinuousTests
         public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
         {
             config = configLoader.Load(args);
-            startupChecker = new StartupChecker(config);
+            startupChecker = new StartupChecker(config, cancelToken);
             this.cancelToken = cancelToken;
         }
 

ContinuousTests/SingleTestRun.cs

@@ -62,7 +62,6 @@ namespace ContinuousTests
         {
             try
             {
-                OverviewLog(" > Starting test. " + FuturesInfo());
                 RunTestMoments();
 
                 if (!config.KeepPassedTestLogs) fixtureLog.Delete();

ContinuousTests/StartupChecker.cs

@@ -1,7 +1,6 @@
 using DistTestCore.Codex;
 using DistTestCore;
 using Logging;
-using NUnit.Framework.Internal;
 
 namespace ContinuousTests
 {
@@ -10,10 +9,12 @@ namespace ContinuousTests
         private readonly TestFactory testFactory = new TestFactory();
         private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
         private readonly Configuration config;
+        private readonly CancellationToken cancelToken;
 
-        public StartupChecker(Configuration config)
+        public StartupChecker(Configuration config, CancellationToken cancelToken)
         {
             this.config = config;
+            this.cancelToken = cancelToken;
         }
 
         public void Check()
@@ -36,6 +37,8 @@ namespace ContinuousTests
             }
             foreach (var test in tests)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 var handle = new TestHandle(test);
                 handle.GetEarliestMoment();
                 handle.GetLastMoment();
@@ -59,6 +62,8 @@ namespace ContinuousTests
             var pass = true;
             foreach (var n in nodes)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 log.Log($"Checking '{n.Address.Host}'...");
 
                 if (EnsureOnline(n))

ContinuousTests/Tests/ThresholdChecks.cs

@@ -0,0 +1,43 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class ThresholdChecks : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        [TestMoment(t: 0)]
+        public void CheckAllThresholds()
+        {
+            var allNodes = CreateAccessToAllNodes();
+            foreach (var n in allNodes) CheckThresholds(n);
+        }
+
+        private void CheckThresholds(CodexAccess n)
+        {
+            var breaches = n.GetDebugThresholdBreaches();
+            if (breaches.breaches.Any())
+            {
+                Assert.Fail(string.Join(",", breaches.breaches.Select(b => FormatBreach(n, b))));
+            }
+        }
+
+        private string FormatBreach(CodexAccess n, string breach)
+        {
+            return $"{n.Container.Name} = '{breach}'";
+        }
+
+        private CodexAccess[] CreateAccessToAllNodes()
+        {
+            // Normally, a continuous test accesses only a subset of the nodes in the deployment.
+            // This time, we want to check all of them.
+            var factory = new CodexAccessFactory();
+            var allContainers = Configuration.CodexDeployment.CodexContainers;
+            return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
+        }
+    }
+}

DistTestCore/Codex/CodexAccess.cs

@@ -53,6 +53,11 @@ namespace DistTestCore.Codex
             return Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
         }
 
+        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
+        {
+            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+        }
+
         public string UploadFile(FileStream fileStream)
         {
             return Http().HttpPostStream("upload", fileStream);

DistTestCore/Codex/CodexApiTypes.cs

@@ -76,6 +76,11 @@ namespace DistTestCore.Codex
         public string address { get; set; } = string.Empty;
     }
 
+    public class CodexDebugThresholdBreaches
+    {
+        public string[] breaches { get; set; } = Array.Empty<string>();
+    }
+
     public class CodexSalesAvailabilityRequest
     {
         public string size { get; set; } = string.Empty;