Sets up some durability tests.

DistTestCore/Codex/CodexStartupConfig.cs

@@ -10,7 +10,6 @@ namespace DistTestCore.Codex
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
-
+        public IOnlineCodexNode? BootstrapNode { get; set; }
-        //public IOnlineCodexNode? BootstrapNode { get; private set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -59,7 +59,7 @@ namespace DistTestCore
 
         public string Describe()
         {
-            return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>";
+            return $"group:[{Containers.Describe()}]";
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

DistTestCore/CodexSetup.cs

@@ -8,7 +8,7 @@ namespace DistTestCore
     {
         ICodexSetup At(Location location);
         ICodexSetup WithLogLevel(CodexLogLevel level);
-        //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
+        ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
@@ -30,7 +30,24 @@ namespace DistTestCore
 
         public ICodexNodeGroup BringOnline()
         {
-            return starter.BringOnline(this);
+            var group = starter.BringOnline(this);
+            ConnectToBootstrapNode(group);
+            return group;
+        }
+
+        private void ConnectToBootstrapNode(ICodexNodeGroup group)
+        {
+            if (BootstrapNode == null) return;
+
+            // TODO:
+            // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection.
+            // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!)
+            // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work.
+            // So for now, ConnectToPeer
+            foreach (var node in group)
+            {
+                node.ConnectToPeer(BootstrapNode);
+            }
         }
 
         public ICodexSetup At(Location location)
@@ -39,11 +56,11 @@ namespace DistTestCore
             return this;
         }
 
-        //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
+        public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
-        //{
+        {
-        //    BootstrapNode = node;
+            BootstrapNode = node;
-        //    return this;
+            return this;
-        //}
+        }
 
         public ICodexSetup WithLogLevel(CodexLogLevel level)
         {
@@ -77,13 +94,13 @@ namespace DistTestCore
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
-            return $"({NumberOfNodes} CodexNodes with [{args}])";
+            return $"({NumberOfNodes} CodexNodes with args:[{args}])";
         }
 
         private IEnumerable<string> DescribeArgs()
         {
             if (LogLevel != null) yield return $"LogLevel={LogLevel}";
-            //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
+            if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}";
             if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }
     }

DistTestCore/CodexStarter.cs

@@ -29,7 +29,7 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: [{string.Join(",", group.Select(n => n.GetName()))}]");
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
             LogSeparator();
             return group;
         }

DistTestCore/OnlineCodexNode.cs

@@ -23,6 +23,7 @@ namespace DistTestCore
         private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly TestLifecycle lifecycle;
+        private CodexDebugResponse? debugInfo;
 
         public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
         {
@@ -45,9 +46,11 @@ namespace DistTestCore
 
         public CodexDebugResponse GetDebugInfo()
         {
-            var response = CodexAccess.GetDebugInfo();
+            if (debugInfo != null) return debugInfo;
-            Log($"Got DebugInfo with id: '{response.id}'.");
+
-            return response;
+            debugInfo = CodexAccess.GetDebugInfo();
+            Log($"Got DebugInfo with id: '{debugInfo.id}'.");
+            return debugInfo;
         }
 
         public ContentId UploadFile(TestFile file)
@@ -89,11 +92,6 @@ namespace DistTestCore
             return lifecycle.DownloadLog(this);
         }
 
-        public string Describe()
-        {
-            return $"({GetName()} in {Group.Describe()})";
-        }
-
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();
@@ -112,8 +110,16 @@ namespace DistTestCore
         private void DownloadToFile(string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);
-            using var downloadStream = CodexAccess.DownloadFile(contentId);
+            try
-            downloadStream.CopyTo(fileStream);
+            {
+                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
         }
 
         private void Log(string msg)

DistTestCore/TestLifecycle.cs

@@ -34,7 +34,7 @@ namespace DistTestCore
         public ICodexNodeLog DownloadLog(OnlineCodexNode node)
         {
             var subFile = Log.CreateSubfile();
-            var description = node.Describe();
+            var description = node.GetName();
             var handler = new LogDownloadHandler(node, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");

KubernetesWorkflow/RunningContainers.cs

@@ -15,7 +15,7 @@
 
         public string Describe()
         {
-            return $"[{RunningPod.Ip}]";
+            return string.Join(",", Containers.Select(c => c.GetName()));
         }
     }
 

Tests/DurabilityTests/DurabilityTests.cs

@@ -0,0 +1,65 @@
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.DurabilityTests
+{
+    [TestFixture]
+    public class DurabilityTests : DistTest
+    {
+        [Test]
+        public void BootstrapNodeDisappearsTest()
+        {
+            var bootstrapNode = SetupCodexNodes(1).BringOnline();
+            var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline();
+            var primary = group[0];
+            var secondary = group[1];
+
+            // There is 1 minute of time for the nodes to connect to each other.
+            // (Should be easy, they're in the same pod.)
+            Time.Sleep(TimeSpan.FromMinutes(1));
+            bootstrapNode.BringOffline();
+
+            var file = GenerateTestFile(10.MB());
+            var contentId = primary.UploadFile(file);
+            var downloadedFile = secondary.DownloadContent(contentId);
+
+            file.AssertIsEqual(downloadedFile);
+        }
+
+        [Test]
+        public void DataRetentionTest()
+        {
+            var bootstrapNode = SetupCodexNodes(1).BringOnline()[0];
+
+            var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline();
+            var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline();
+
+            var file = GenerateTestFile(10.MB());
+
+            // Both nodes in the start group have the file.
+            var content = startGroup[0].UploadFile(file);
+            DownloadAndAssert(content, file, startGroup[1]);
+
+            // Three nodes of the finish group have the file.
+            DownloadAndAssert(content, file, finishGroup[0]);
+            DownloadAndAssert(content, file, finishGroup[1]);
+            DownloadAndAssert(content, file, finishGroup[2]);
+
+            // The start group goes away.
+            startGroup.BringOffline();
+
+            // All nodes in the finish group can access the file.
+            foreach (var node in finishGroup)
+            {
+                DownloadAndAssert(content, file, node);
+            }
+        }
+
+        private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode)
+        {
+            var downloaded = onlineCodexNode.DownloadContent(content);
+            file.AssertIsEqual(downloaded);
+        }
+    }
+}