Moves log downloader into continous test runner

CodexNetDownloader/Configuration.cs

@@ -10,16 +10,13 @@ namespace CodexNetDownloader
         public string OutputPath { get; set; } = "output";
 
         [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
-        public string CodexDeploymentJson { get; set; } = string.Empty;
+        public string CodexDeploymentJson { get; set; } = string.Empty; // @"d:\Projects\cs-codex-dist-tests\CodexNetDownloader\codex-deployment.json";
 
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
-        public string KubeConfigFile { get; set; } = "null";
-
-        [Uniform("continuous", "c", "CONTINUOUS", false, "If true, will continuously download and append log files.")]
-        public bool Continuous { get; set; } = false;
+        public string KubeConfigFile { get; set; } = string.Empty;// @"c:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.ExternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using CodexNetDownloader;
 using ContinuousTests;
 using DistTestCore;
 using DistTestCore.Codex;
@@ -28,13 +27,6 @@ public class Program
         var k8sFactory = new K8sFactory();
         var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
 
-        if (config.Continuous)
-        {
-            var dl = new ContinuousLogDownloader(lifecycle, config);
-            dl.Run();
-        }
-        else
-        {
         foreach (var container in config.CodexDeployment.CodexContainers)
         {
             lifecycle.DownloadLog(container);
@@ -42,7 +34,6 @@ public class Program
 
         Console.WriteLine("Done!");
     }
-    }
 
     private static CodexDeployment ParseCodexDeploymentJson(string filename)
     {

ContinuousTests/Configuration.cs

@@ -25,6 +25,9 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
         public bool StopOnFailure { get; set; } = false;
 
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;

ContinuousTests/ContinuousLogDownloader.cs

@@ -1,33 +1,37 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using KubernetesWorkflow;
 
-namespace CodexNetDownloader
+namespace ContinuousTests
 {
     public class ContinuousLogDownloader
     {
         private readonly TestLifecycle lifecycle;
-        private readonly Configuration config;
+        private readonly CodexDeployment deployment;
+        private readonly string outputPath;
+        private readonly CancellationToken cancelToken;
 
-        public ContinuousLogDownloader(TestLifecycle lifecycle, Configuration config)
+        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
         {
             this.lifecycle = lifecycle;
-            this.config = config;
+            this.deployment = deployment;
+            this.outputPath = outputPath;
+            this.cancelToken = cancelToken;
         }
 
         public void Run()
         {
-            while (true)
+            while (!cancelToken.IsCancellationRequested)
             {
                 UpdateLogs();
 
-                Thread.Sleep(TimeSpan.FromSeconds(30));
+                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
             }
         }
 
         private void UpdateLogs()
         {
-            Console.WriteLine("Updating logs...");
-            foreach (var container in config.CodexDeployment.CodexContainers)
+            foreach (var container in deployment.CodexContainers)
             {
                 UpdateLog(container);
             }
@@ -35,8 +39,11 @@ namespace CodexNetDownloader
 
         private void UpdateLog(RunningContainer container)
         {
-            var filepath = Path.Combine(config.OutputPath, GetLogName(container));
-            if (!File.Exists(filepath)) File.WriteAllLines(filepath, new[] { "" });
+            var filepath = Path.Combine(outputPath, GetLogName(container));
+            if (!File.Exists(filepath))
+            {
+                File.WriteAllLines(filepath, new[] { container.Name });
+            }
 
             var appender = new LogAppender(filepath);
 
@@ -46,7 +53,7 @@ namespace CodexNetDownloader
         private static string GetLogName(RunningContainer container)
         {
             return container.Name
-                .Replace("<","")
+                .Replace("<", "")
                 .Replace(">", "")
                 + ".log";
         }
@@ -64,21 +71,23 @@ namespace CodexNetDownloader
         public void Log(Stream log)
         {
             using var reader = new StreamReader(log);
-            var currentLines = File.ReadAllLines(filename);
+            var lines = File.ReadAllLines(filename);
+            var lastLine = lines.Last();
+            var recording = lines.Length < 3;
             var line = reader.ReadLine();
             while (line != null)
             {
-                AppendLineIfNew(line, currentLines);
-                line = reader.ReadLine();
-            }
-        }
-
-        private void AppendLineIfNew(string line, string[] currentLines)
-        {
-            if (!currentLines.Contains(line))
+                if (recording)
                 {
                     File.AppendAllLines(filename, new[] { line });
                 }
+                else
+                {
+                    recording = line == lastLine;
+                }
+
+                line = reader.ReadLine();
+            }
         }
     }
 }

ContinuousTests/ContinuousTestRunner.cs

@@ -30,6 +30,8 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
+            StartLogDownloader(taskFactory);
+
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
@@ -61,5 +63,18 @@ namespace ContinuousTests
             var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
             workflowCreator.CreateWorkflow().DeleteTestResources();
         }
+
+        private void StartLogDownloader(TaskFactory taskFactory)
+        {
+            if (!config.DownloadContainerLogs) return;
+
+            var path = Path.Combine(config.LogPath, "containers");
+            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
+
+            var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+
+            taskFactory.Run(downloader.Run);
+        }
     }
 }

ContinuousTests/run.sh

@@ -2,4 +2,5 @@ dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
-    --stop=1 
+    --stop=1 \
+    --dl-logs=1