Allows downloading only log-tails.

ContinuousTests/ContinuousLogDownloader.cs

@@ -55,7 +55,7 @@ namespace ContinuousTests
 
             var appender = new LogAppender(filepath);
 
-            lifecycle.CodexStarter.DownloadLog(container, appender);
+            lifecycle.CodexStarter.DownloadLog(container, appender, null);
         }
 
         private static string GetLogName(RunningContainer container)

DistTestCore/CodexStarter.cs

@@ -64,10 +64,10 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
         }
 
         private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler);
+                workflow.DownloadContainerLog(container, logHandler, null);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder);
+            workflow.DownloadContainerLog(container, enodeFinder, null);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/OnlineCodexNode.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog();
+        IDownloadedLog DownloadLog(int? tailLines = null);
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         CodexDebugVersionResponse Version { get; }
@@ -103,9 +103,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog()
+        public IDownloadedLog DownloadLog(int? tailLines = null)
         {
-            return lifecycle.DownloadLog(CodexAccess.Container);
+            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
         public ICodexSetup BringOffline()

DistTestCore/PrometheusStarter.cs

@@ -29,7 +29,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 30s\n";
+            config += "  scrape_interval: 10s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -49,14 +49,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container)
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler);
+            CodexStarter.DownloadLog(container, handler, tailLines);
 
             return new DownloadedLog(subFile, description);
         }

KubernetesWorkflow/K8sController.cs

@@ -53,10 +53,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
             logHandler.Log(stream);
         }
 

KubernetesWorkflow/StartupWorkflow.cs

@@ -50,11 +50,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
             });
         }
 

Tests/BasicTests/ContinuousSubstitute.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using Logging;
 using NUnit.Framework;
 using Utils;
 
@@ -61,12 +62,13 @@ namespace Tests.BasicTests
         [Test]
         public void HoldMyBeerTest()
         {
-            var group = SetupCodexNodes(5, o => o
+            var blockExpirationTime = TimeSpan.FromMinutes(3);
+            var group = SetupCodexNodes(1, o => o
                     .EnableMetrics()
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithBlockTTL(blockExpirationTime)
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(5))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(1))
                     .WithBlockMaintenanceNumber(10000)
-                    .WithStorageQuota(1000.MB()));
+                    .WithStorageQuota(2000.MB()));
 
             var nodes = group.Cast<OnlineCodexNode>().ToArray();
 
@@ -78,43 +80,63 @@ namespace Tests.BasicTests
             var sizeInBytes = filesize.SizeInBytes;
             Assert.That(numberOfBlocks, Is.EqualTo(1282));
 
+            var successfulUploads = 0;
+            var successfulDownloads = 0;
+
             while (DateTime.UtcNow < endTime)
             {
                 foreach (var node in nodes)
                 {
                     try
                     {
+                        var uploadStartTime = DateTime.UtcNow;
                         var file = GenerateTestFile(filesize);
                         var cid = node.UploadFile(file);
 
                         var cidTag = cid.Id.Substring(cid.Id.Length - 6);
-                        var uploadLog = node.DownloadLog();
+                        Stopwatch.Measure(Get().Log, "upload-log-asserts", () =>
+                        {
+                            var uploadLog = node.DownloadLog(tailLines: 50000);
 
-                        var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                            var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
-                        uploadLog.DeleteFile();
+                            uploadLog.DeleteFile();
 
-                        var storeLine = GetLineForCidTag(storeLines, cidTag);
+                            var storeLine = GetLineForCidTag(storeLines, cidTag);
-                        AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                            AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                        });
+                        successfulUploads++;
+
+                        var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
+                        if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
+                        {
+                            Assert.Fail("Upload took too long. Blocks already expired.");
+                        }
 
                         var dl = node.DownloadContent(cid);
                         file.AssertIsEqual(dl);
-                        var downloadLog = node.DownloadLog();
 
-                        var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                        Stopwatch.Measure(Get().Log, "download-log-asserts", () =>
-                        downloadLog.DeleteFile();
+                        {
+                            var downloadLog = node.DownloadLog(tailLines: 50000);
 
-                        var sentLine = GetLineForCidTag(sentLines, cidTag);
+                            var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
-                        AssertSentLineContains(sentLine, sizeInBytes);
+                            downloadLog.DeleteFile();
+
+                            var sentLine = GetLineForCidTag(sentLines, cidTag);
+                            AssertSentLineContains(sentLine, sizeInBytes);
+                        });
+                        successfulDownloads++;
                     }
                     catch
                     {
                         Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
+                        Log($"Test failed after {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
                         Thread.Sleep(TimeSpan.FromSeconds(30));
                         throw;
                     }
                 }
 
-                Thread.Sleep(TimeSpan.FromSeconds(3));
+                Thread.Sleep(TimeSpan.FromSeconds(5));
             }
         }