rigged up a pod crash detector maybe

DistTestCore/Metrics/dashboard.json

@@ -377,7 +377,8 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "decbytes"
         },
         "overrides": []
       },
@@ -471,7 +472,8 @@
                 "value": "<CODEX_STORAGEQUOTA>"
               }
             ]
-          }
+          },
+          "unit": "decbytes"
         },
         "overrides": []
       },

KubernetesWorkflow/K8sController.cs

@@ -604,6 +604,30 @@ namespace KubernetesWorkflow
 
         #endregion
 
+        public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler)
+        {
+            return Task.Run(() =>
+            {
+                var myOwnClient = new Kubernetes(cluster.GetK8sClientConfig());
+                while (!token.IsCancellationRequested)
+                {
+                    token.WaitHandle.WaitOne(TimeSpan.FromSeconds(3));
+
+                    var pod = container.Pod;
+                    var recipe = container.Recipe;
+                    var podName = pod.PodInfo.Name;
+                    var podInfo = myOwnClient.ReadNamespacedPod(podName, K8sTestNamespace);
+                    
+                    if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0))
+                    {
+                        log.Log("Pod crash detected for " + container.Name);
+                        using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, previous: true));
+                        logHandler.Log(stream);
+                    }
+                }
+            });
+        }
+
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/StartupWorkflow.cs

@@ -37,6 +37,11 @@ namespace KubernetesWorkflow
             }, pl);
         }
 
+        public Task WatchForCrashLogs(RunningContainer container, CancellationToken token, ILogHandler logHandler)
+        {
+            return K8s(controller => controller.WatchForCrashLogs(container, token, logHandler));
+        }
+
         public void Stop(RunningContainers runningContainers)
         {
             K8s(controller =>

Tests/BasicTests/ContinuousSubstitute.cs

@@ -1,11 +1,12 @@
 using DistTestCore;
+using KubernetesWorkflow;
 using NUnit.Framework;
 using Utils;
 
 namespace Tests.BasicTests
 {
     [TestFixture]
-    public class ContinuousSubstitute : AutoBootstrapDistTest
+    public class ContinuousSubstitute : AutoBootstrapDistTest, ILogHandler
     {
         [Test]
         [UseLongTimeouts]
@@ -70,6 +71,12 @@ namespace Tests.BasicTests
 
             var nodes = group.Cast<OnlineCodexNode>().ToArray();
 
+            var flow = Get().WorkflowCreator.CreateWorkflow();
+            var cst = new CancellationTokenSource();
+            var tasks = nodes.Select(n => flow.WatchForCrashLogs(n.CodexAccess.Container, cst.Token, this)).ToArray();
+
+            try
+            {
                 foreach (var node in nodes)
                 {
                     node.Marketplace.MakeStorageAvailable(
@@ -94,5 +101,24 @@ namespace Tests.BasicTests
                     Thread.Sleep(TimeSpan.FromMinutes(2));
                 }
             }
+            finally
+            {
+                cst.Cancel();
+                foreach (var t in tasks) t.Wait();
+            }
+        }
+
+        public void Log(Stream log)
+        {
+            Log("Well damn, container crashed. Here's the log:");
+            using var reader = new StreamReader(log);
+
+            var line = reader.ReadLine();
+            while(line != null)
+            {
+                Log(line);
+                line = reader.ReadLine();
+            }
+        }
     }
 }