From a1ec187919defeb14301e9aedc6d497593343150 Mon Sep 17 00:00:00 2001
From: benbierens <thatbenbierens@gmail.com>
Date: Wed, 30 Aug 2023 09:23:13 +0200
Subject: [PATCH] Configured runner to stop after 10 failures

---
 ContinuousTests/Configuration.cs |  4 ++--
 ContinuousTests/SingleTestRun.cs | 16 +++++++++++-----
 ContinuousTests/run.sh           |  2 +-
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs
index da1eaeb..04491a9 100644
--- a/ContinuousTests/Configuration.cs
+++ b/ContinuousTests/Configuration.cs
@@ -22,8 +22,8 @@ namespace ContinuousTests
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
-        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
-        public bool StopOnFailure { get; set; } = false;
+        [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
+        public int StopOnFailure { get; set; } = 0;
 
         [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
         public bool DownloadContainerLogs { get; set; } = false;
diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs
index e268064..a43dc2f 100644
--- a/ContinuousTests/SingleTestRun.cs
+++ b/ContinuousTests/SingleTestRun.cs
@@ -23,6 +23,7 @@ namespace ContinuousTests
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
+        private static int failureCount = 0;
 
         public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
         {
@@ -71,12 +72,17 @@ namespace ContinuousTests
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
 
-                if (config.StopOnFailure)
+                failureCount++;
+                if (config.StopOnFailure > 0)
                 {
-                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
-                    DownloadClusterLogs();
-                    OverviewLog("Log download finished. Cancelling test runner...");
-                    Cancellation.Cts.Cancel();
+                    OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
+                    if (failureCount >= config.StopOnFailure)
+                    {
+                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
+                        DownloadClusterLogs();
+                        OverviewLog("Log download finished. Cancelling test runner...");
+                        Cancellation.Cts.Cancel();
+                    }
                 }
             }
         }
diff --git a/ContinuousTests/run.sh b/ContinuousTests/run.sh
index 67c9419..6b281d7 100644
--- a/ContinuousTests/run.sh
+++ b/ContinuousTests/run.sh
@@ -2,5 +2,5 @@ dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
-    --stop=1 \
+    --stop=10 \
     --dl-logs=1