Debuginfo test and One-client test are working

BasicTests/DebugEndpointTests.cs

@@ -15,16 +15,34 @@ namespace CodexDistTests.BasicTests
 
             Assert.That(debugInfo.spr, Is.Not.Empty);
         }
-        
+
+        [Test]
+        public void OneClientTest()
+        {
+            var primary = SetupCodexNode()
+                                .WithLogLevel(CodexLogLevel.Trace)
+                                .WithStorageQuota(1024 * 1024 * 2)
+                                .BringOnline();
+
+            var testFile = GenerateTestFile(1024 * 1024);
+
+            var contentId = primary.UploadFile(testFile);
+
+            var downloadedFile = primary.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+
         //[Test]
         //public void TwoClientTest()
         //{
         //    var primary = SetupCodexNode()
-        //                        .WithLogLevel(CodexLogLevel.Warn)
-        //                        .WithStorageQuota(1024 * 1024)
+        //                        .WithLogLevel(CodexLogLevel.Trace)
+        //                        .WithStorageQuota(1024 * 1024 * 2)
         //                        .BringOnline();
 
         //    var secondary = SetupCodexNode()
+        //                        .WithLogLevel(CodexLogLevel.Trace)
         //                        .WithBootstrapNode(primary)
         //                        .BringOnline();
 

TestCore/K8sManager.cs

@@ -20,6 +20,7 @@ namespace CodexDistTests.TestCore
 
         private V1Namespace? activeNamespace;
         private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
+        private readonly List<string> knownActivePodNames = new List<string>();
 
         public K8sManager(IFileManager fileManager)
         {
@@ -63,11 +64,6 @@ namespace CodexDistTests.TestCore
         {
             var client = CreateClient();
 
-            foreach (var activeNode in activeNodes.Values)
-            {
-                BringOffline(activeNode, client);
-            }
-
             DeleteNamespace(client);
 
             WaitUntilZeroPods(client);
@@ -76,6 +72,7 @@ namespace CodexDistTests.TestCore
 
         private void BringOffline(ActiveNode activeNode, Kubernetes client)
         {
+            DownloadCodexNodeLog(activeNode, client);
             DeleteDeployment(activeNode, client);
             DeleteService(activeNode, client);
         }
@@ -89,6 +86,22 @@ namespace CodexDistTests.TestCore
                 activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace);
                 return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0;
             });
+
+            AssignActivePodNames(activeNode, client);
+        }
+
+        private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client)
+        {
+            var pods = client.ListNamespacedPod(k8sNamespace);
+            var podNames = pods.Items.Select(p => p.Name());
+            foreach (var podName in podNames)
+            {
+                if (!knownActivePodNames.Contains(podName))
+                {
+                    knownActivePodNames.Add(podName);
+                    activeNode.ActivePodNames.Add(podName);
+                }
+            }
         }
 
         private void WaitUntilOffline(string deploymentName, Kubernetes client)
@@ -245,6 +258,7 @@ namespace CodexDistTests.TestCore
         {
             if (activeNamespace == null) return;
             client.DeleteNamespace(activeNamespace.Name());
+            activeNamespace = null;
         }
 
         #endregion
@@ -256,6 +270,21 @@ namespace CodexDistTests.TestCore
             return new Kubernetes(config);
         }
 
+        private void DownloadCodexNodeLog(ActiveNode node, Kubernetes client)
+        {
+            //var client = CreateClient();
+            var i = 0;
+            foreach (var podName in node.ActivePodNames)
+            {
+                var stream = client.ReadNamespacedPodLog(podName, k8sNamespace);
+                using (var fileStream = File.Create(node.SelectorName + i.ToString() + ".txt"))
+                {
+                    stream.CopyTo(fileStream);
+                }
+                i++;
+            }
+        }
+
         private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
         {
             var n = (OnlineCodexNode)node;
@@ -292,6 +321,7 @@ namespace CodexDistTests.TestCore
             public int Port { get; }
             public V1Deployment? Deployment { get; set; }
             public V1Service? Service { get; set; }
+            public List<string> ActivePodNames { get; } = new List<string>();
 
             public V1ObjectMeta GetServiceMetadata()
             {

TestCore/OnlineCodexNode.cs

@@ -49,6 +49,11 @@ namespace CodexDistTests.TestCore
                 var response = Utils.Wait(client.PostAsync(url, content));
 
                 var contentId = Utils.Wait(response.Content.ReadAsStringAsync());
+                if (contentId.StartsWith("Unable to store block"))
+                {
+                    retryCounter = Timing.HttpCallRetryCount() + 1;
+                    Assert.Fail("Node failed to store block.");
+                }
                 return new ContentId(contentId);
             }
             catch (Exception exception)