Merge branch 'feature/log-continuous-downloader'

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -9,4 +9,5 @@ dotnet run \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=120
+    --block-ttl=120 \
+	-y

ContinuousTests/Configuration.cs

@@ -25,6 +25,9 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
         public bool StopOnFailure { get; set; } = false;
 
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;

ContinuousTests/ContinuousLogDownloader.cs

@@ -0,0 +1,99 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+    public class ContinuousLogDownloader
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly CodexDeployment deployment;
+        private readonly string outputPath;
+        private readonly CancellationToken cancelToken;
+
+        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
+        {
+            this.lifecycle = lifecycle;
+            this.deployment = deployment;
+            this.outputPath = outputPath;
+            this.cancelToken = cancelToken;
+        }
+
+        public void Run()
+        {
+            while (!cancelToken.IsCancellationRequested)
+            {
+                UpdateLogs();
+
+                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
+            }
+
+            // After testing has stopped, we wait a little bit and fetch the logs one more time.
+            // If our latest fetch was not recent, interesting test-related log activity might
+            // not have been captured yet.
+            Thread.Sleep(TimeSpan.FromSeconds(10));
+            UpdateLogs();
+        }
+
+        private void UpdateLogs()
+        {
+            foreach (var container in deployment.CodexContainers)
+            {
+                UpdateLog(container);
+            }
+        }
+
+        private void UpdateLog(RunningContainer container)
+        {
+            var filepath = Path.Combine(outputPath, GetLogName(container));
+            if (!File.Exists(filepath))
+            {
+                File.WriteAllLines(filepath, new[] { container.Name });
+            }
+
+            var appender = new LogAppender(filepath);
+
+            lifecycle.CodexStarter.DownloadLog(container, appender);
+        }
+
+        private static string GetLogName(RunningContainer container)
+        {
+            return container.Name
+                .Replace("<", "")
+                .Replace(">", "")
+                + ".log";
+        }
+    }
+
+    public class LogAppender : ILogHandler
+    {
+        private readonly string filename;
+
+        public LogAppender(string filename)
+        {
+            this.filename = filename;
+        }
+
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var lines = File.ReadAllLines(filename);
+            var lastLine = lines.Last();
+            var recording = lines.Length < 3;
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                if (recording)
+                {
+                    File.AppendAllLines(filename, new[] { line });
+                }
+                else
+                {
+                    recording = line == lastLine;
+                }
+
+                line = reader.ReadLine();
+            }
+        }
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -30,6 +30,8 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
+            StartLogDownloader(taskFactory);
+
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
@@ -61,5 +63,18 @@ namespace ContinuousTests
             var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
             workflowCreator.CreateWorkflow().DeleteTestResources();
         }
+
+        private void StartLogDownloader(TaskFactory taskFactory)
+        {
+            if (!config.DownloadContainerLogs) return;
+
+            var path = Path.Combine(config.LogPath, "containers");
+            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
+
+            var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+
+            taskFactory.Run(downloader.Run);
+        }
     }
 }

ContinuousTests/run.sh

@@ -2,4 +2,5 @@ dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
-    --stop=1 
+    --stop=1 \
+    --dl-logs=1