Good progress

CodexPlugin/CodexAccess.cs

@@ -7,11 +7,11 @@ namespace CodexPlugin
 {
     public class CodexAccess : ILogHandler
     {
-        private readonly BaseLog log;
+        private readonly ILog log;
         private readonly ITimeSet timeSet;
         private bool hasContainerCrashed;
 
-        public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
+        public CodexAccess(ILog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
             this.log = log;
             Container = container;

CodexPlugin/CodexNodeFactory.cs

@@ -3,7 +3,7 @@ namespace CodexPlugin
 {
     public interface ICodexNodeFactory
     {
-        //OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
+        OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
     }
 
     public class CodexNodeFactory : ICodexNodeFactory
@@ -19,11 +19,11 @@ namespace CodexPlugin
         //    this.marketplaceAccessFactory = marketplaceAccessFactory;
         //}
 
-        //public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
+        public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        //{
+        {
-        //    var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
+            //var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
-        //    var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
+            //var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
-        //    return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
+            return new OnlineCodexNode(/*lifecycle,*/ access, group/*, metricsAccess, marketplaceAccess*/);
-        //}
+        }
     }
 }

CodexPlugin/CodexNodeGroup.cs

@@ -1,11 +1,13 @@
-using KubernetesWorkflow;
+using DistTestCore;
+using KubernetesWorkflow;
+using Logging;
 using System.Collections;
 
 namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
     {
-        ICodexSetup BringOffline();
+        void BringOffline();
         IOnlineCodexNode this[int index] { get; }
     }
 
@@ -13,12 +15,12 @@ namespace CodexPlugin
     {
         //private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(/*TestLifecycle lifecycle, */CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(/*TestLifecycle lifecycle, CodexSetup setup,*/ILog log, ITimeSet timeSet, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             //this.lifecycle = lifecycle;
-            Setup = setup;
+            //Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, log, timeSet, codexNodeFactory)).ToArray();
             Version = new CodexDebugVersionResponse();
         }
 
@@ -30,20 +32,18 @@ namespace CodexPlugin
             }
         }
 
-        public ICodexSetup BringOffline()
+        public void BringOffline()
         {
             //lifecycle.CodexStarter.BringOffline(this);
 
-            var result = Setup;
+            //var result = Setup;
             // Clear everything. Prevent accidental use.
-            Setup = null!;
+            //Setup = null!;
             Nodes = Array.Empty<OnlineCodexNode>();
             Containers = null!;
-
-            return result;
         }
 
-        public CodexSetup Setup { get; private set; }
+        //public CodexSetup Setup { get; private set; }
         public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
         public CodexDebugVersionResponse Version { get; private set; }
@@ -78,11 +78,10 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
+        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ILog log, ITimeSet timeSet, ICodexNodeFactory factory)
         {
-            //var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
+            var access = new CodexAccess(log, c, timeSet, c.Address);
-            //return factory.CreateOnlineCodexNode(access, this);
+            return factory.CreateOnlineCodexNode(access, this);
-            return null!;
         }
     }
 }

CodexPlugin/CodexStarter.cs

@@ -1,26 +1,32 @@
-using KubernetesWorkflow;
+using DistTestCore;
+using KubernetesWorkflow;
 using Logging;
 
 namespace CodexPlugin
 {
-    public class CodexStarter //: BaseStarter
+    public class CodexStarter
     {
+        private readonly IPluginActions pluginActions;
+
         //public CodexStarter(TestLifecycle lifecycle)
         //    : base(lifecycle)
         //{
         //}
 
-        public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
+        public CodexStarter(IPluginActions pluginActions)
+        {
+            this.pluginActions = pluginActions;
+        }
 
-        public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
+        public RunningContainers[] BringOnline(CodexSetup codexSetup)
         {
             //LogSeparator();
             //LogStart($"Starting {codexSetup.Describe()}...");
             //var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
 
-            //var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
+            var startupConfig = CreateStartupConfig(/*gethStartResult,*/ codexSetup);
 
-            //var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
+            return StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
             //var metricAccessFactory = CollectMetrics(codexSetup, containers);
 
@@ -38,7 +44,26 @@ namespace CodexPlugin
             //LogSeparator();
 
             //return group;
-            return null!;
+        }
+
+        public ICodexNodeGroup WrapCodexContainers(RunningContainers[] containers)
+        {
+            //var metricAccessFactory = CollectMetrics(codexSetup, containers);
+
+            var codexNodeFactory = new CodexNodeFactory();// (lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
+
+            return CreateCodexGroup(/*codexSetup,*/ containers, codexNodeFactory);
+            //lifecycle.SetCodexVersion(group.Version);
+
+            //var nl = Environment.NewLine;
+            //var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
+            //LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
+            //    $"of image '{containers.Containers().First().Recipe.Image}' " +
+            //    $"and version '{group.Version}'{nl}" +
+            //    podInfos);
+            //LogSeparator();
+
+            //return group;
         }
 
         public void BringOffline(CodexNodeGroup group)
@@ -50,7 +75,6 @@ namespace CodexPlugin
             //    StopCrashWatcher(c);
             //    workflow.Stop(c);
             //}
-            //RunningGroups.Remove(group);
             //LogEnd("Stopped.");
         }
 
@@ -58,8 +82,6 @@ namespace CodexPlugin
         {
             //var workflow = CreateWorkflow();
             //workflow.DeleteTestResources();
-
-            //RunningGroups.Clear();
         }
 
         public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
@@ -82,52 +104,51 @@ namespace CodexPlugin
         //    return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         //}
 
-        //private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
+        private StartupConfig CreateStartupConfig(/*GethStartResult gethStartResult, */ CodexSetup codexSetup)
-        //{
+        {
-        //    var startupConfig = new StartupConfig();
+            var startupConfig = new StartupConfig();
-        //    startupConfig.NameOverride = codexSetup.NameOverride;
+            startupConfig.NameOverride = codexSetup.NameOverride;
-        //    startupConfig.Add(codexSetup);
+            startupConfig.CreateCrashWatcher = true;
-        //    startupConfig.Add(gethStartResult);
+            startupConfig.Add(codexSetup);
-        //    return startupConfig;
+            //startupConfig.Add(gethStartResult);
-        //}
+            return startupConfig;
+        }
 
-        //private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
-        //{
+        {
-        //    var result = new List<RunningContainers>();
+            var result = new List<RunningContainers>();
-        //    var recipe = new CodexContainerRecipe();
+            var recipe = new CodexContainerRecipe();
-        //    for (var i = 0; i < numberOfNodes; i++)
+            for (var i = 0; i < numberOfNodes; i++)
-        //    {
+            {
-        //        var workflow = CreateWorkflow();
+                var workflow = pluginActions.CreateWorkflow();
-        //        var rc = workflow.Start(1, location, recipe, startupConfig);
+                result.Add(workflow.Start(1, location, recipe, startupConfig));
-        //        CreateCrashWatcher(workflow, rc);
+            }
-        //        result.Add(rc);
+            return result.ToArray();
-        //    }
+        }
-        //    return result.ToArray();
-        //}
 
-        //private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(/*CodexSetup codexSetup, */RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
-        //{
+        {
-        //    var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
+            var group = new CodexNodeGroup(pluginActions.GetLog(), pluginActions.GetTimeSet(), /*lifecycle, codexSetup,*/ runningContainers, codexNodeFactory);
-        //    RunningGroups.Add(group);
 
-        //    try
+            try
-        //    {
+            {
-        //        Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+                Stopwatch.Measure(pluginActions.GetLog(), "EnsureOnline", group.EnsureOnline, debug: true);
-        //    }
+            }
-        //    catch
+            catch
-        //    {
+            {
-        //        CodexNodesNotOnline(runningContainers);
+                CodexNodesNotOnline(runningContainers);
-        //        throw;
+                throw;
-        //    }
+            }
 
-        //    return group;
+            return group;
-        //}
+        }
 
-        //private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
-        //{
+        {
-        //    Log("Codex nodes failed to start");
+            pluginActions.GetLog().Log("Codex nodes failed to start");
-        //    foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+            // todo:
-        //}
+            //foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
 
         //private StartupWorkflow CreateWorkflow()
         //{
@@ -139,12 +160,6 @@ namespace CodexPlugin
         //    Log("----------------------------------------------------------------------------");
         //}
 
-        //private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
-        //{
-        //    var c = rc.Containers.Single();
-        //    c.CrashWatcher = workflow.CreateCrashWatcher(c);
-        //}
-
         //private void StopCrashWatcher(RunningContainers containers)
         //{
         //    foreach (var c in containers.Containers)

CodexPlugin/DistTestExtensions.cs

@@ -5,24 +5,26 @@ namespace CodexPlugin
 {
     public static class DistTestExtensions
     {
-        public static RunningContainers StartCodexNodes(this DistTest distTest, int number, Action<ICodexSetup> setup)
+        public static Plugin Plugin { get; internal set; } = null!;
+
+        public static RunningContainers[] StartCodexNodes(this DistTest distTest, int number, Action<ICodexSetup> setup)
         {
-            return null!;
+            return Plugin.StartCodexNodes(number, setup);
         }
 
         public static ICodexNodeGroup WrapCodexContainers(this DistTest distTest, RunningContainers containers)
         {
-            return null!;
+            return Plugin.WrapCodexContainers(containers);
         }
 
         public static IOnlineCodexNode SetupCodexNode(this DistTest distTest, Action<ICodexSetup> setup)
         {
-            return null!;
+            return Plugin.SetupCodexNode(setup);
         }
 
         public static ICodexNodeGroup SetupCodexNodes(this DistTest distTest, int number)
         {
-            return null!;
+            return Plugin.SetupCodexNodes(number);
         }
     }
 }

CodexPlugin/OnlineCodexNode.cs

@@ -1,6 +1,5 @@
 using DistTestCore.Logs;
 using FileUtils;
-using Logging;
 using NUnit.Framework;
 using Utils;
 
@@ -18,7 +17,7 @@ namespace CodexPlugin
         //IMetricsAccess Metrics { get; }
         //IMarketplaceAccess Marketplace { get; }
         CodexDebugVersionResponse Version { get; }
-        ICodexSetup BringOffline();
+        void BringOffline();
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
@@ -108,13 +107,13 @@ namespace CodexPlugin
             return null!; // lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
-        public ICodexSetup BringOffline()
+        public void BringOffline()
         {
             if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
                 "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
                 "available for codex-nodes in groups of 1.");
 
-            return Group.BringOffline();
+            Group.BringOffline();
         }
 
         public void EnsureOnlineGetVersionResponse()

CodexPlugin/Plugin.cs

@@ -0,0 +1,40 @@
+using DistTestCore;
+using KubernetesWorkflow;
+
+namespace CodexPlugin
+{
+    public class Plugin : IProjectPlugin
+    {
+        private readonly CodexStarter codexStarter;
+
+        public Plugin(IPluginActions actions)
+        {
+            codexStarter = new CodexStarter(actions);
+
+            DistTestExtensions.Plugin = this;
+        }
+
+        public RunningContainers[] StartCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = new CodexSetup(numberOfNodes, CodexLogLevel.Trace);
+            setup(codexSetup);
+            return codexStarter.BringOnline(codexSetup);
+        }
+
+        public ICodexNodeGroup WrapCodexContainers(RunningContainers[] containers)
+        {
+            return codexStarter.WrapCodexContainers(containers);
+        }
+
+        public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
+        {
+            return null!;
+        }
+
+        public ICodexNodeGroup SetupCodexNodes(int number)
+        {
+            var rc = StartCodexNodes(1, s => { });
+            return WrapCodexContainers(rc);
+        }
+    }
+}

DistTestCore/Configuration.cs

@@ -12,7 +12,6 @@ namespace DistTestCore
         private readonly string dataFilesPath;
         //private readonly CodexLogLevel codexLogLevel;
         private readonly string k8sNamespacePrefix;
-        private static RunnerLocation? runnerLocation = null;
 
         public Configuration()
         {
@@ -59,20 +58,6 @@ namespace DistTestCore
         //    return codexLogLevel;
         //}
 
-        public Address GetAddress(RunningContainer container)
-        {
-            if (runnerLocation == null)
-            {
-                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
-            }
-            
-            if (runnerLocation == RunnerLocation.InternalToCluster)
-            {
-                return container.ClusterInternalAddress;
-            }
-            return container.ClusterExternalAddress;
-        }
-
         private static string GetEnvVarOrDefault(string varName, string defaultValue)
         {
             var v = Environment.GetEnvironmentVariable(varName);
@@ -88,56 +73,5 @@ namespace DistTestCore
         }
     }
 
-    public enum RunnerLocation
-    {
-        ExternalToCluster,
-        InternalToCluster,
-    }
 
-    public static class RunnerLocationUtils
-    {
-        private static bool alreadyDidThat = false;
-
-        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
-        {
-            // We want to be sure we don't ping more often than strictly necessary.
-            // If we have already determined the location during this application
-            // lifetime, don't do it again.
-            if (alreadyDidThat) throw new Exception("We already did that.");
-            alreadyDidThat = true;
-
-            if (PingHost(container.Pod.PodInfo.Ip))
-            {
-                return RunnerLocation.InternalToCluster;
-            }
-            if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                return RunnerLocation.ExternalToCluster;
-            }
-
-            throw new Exception("Unable to determine runner location.");
-        }
-
-        private static string Format(Address host)
-        {
-            return host.Host
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
-
-        private static bool PingHost(string host)
-        {
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
-
-            return false;
-        }
-    }
 }

DistTestCore/Http.cs

@@ -9,19 +9,19 @@ namespace DistTestCore
 {
     public class Http
     {
-        private readonly BaseLog log;
+        private readonly ILog log;
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+        public Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
             : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
         {
         }
 
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;

DistTestCore/PluginManager.cs

@@ -0,0 +1,64 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public class PluginManager : IPluginActions
+    {
+        private readonly BaseLog log;
+        private readonly Configuration configuration;
+        private readonly string testNamespace;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly ITimeSet timeSet;
+        private readonly List<IProjectPlugin> projectPlugins = new List<IProjectPlugin>();
+
+        public PluginManager(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
+        {
+            this.log = log;
+            this.configuration = configuration;
+            this.timeSet = timeSet;
+            this.testNamespace = testNamespace;
+            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
+        }
+
+        public IStartupWorkflow CreateWorkflow()
+        {
+            return workflowCreator.CreateWorkflow();
+        }
+
+        public ILog GetLog()
+        {
+            return log;
+        }
+
+        public ITimeSet GetTimeSet()
+        {
+            return timeSet;
+        }
+
+        public void InitializeAllPlugins()
+        {
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            var pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t => typeof(IProjectPlugin).IsAssignableFrom(t))).ToArray();
+
+            foreach (var pluginType in pluginTypes)
+            {
+                IPluginActions actions = this;
+                var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: actions)!;
+                projectPlugins.Add(plugin);
+            }
+        }
+    }
+
+    public interface IProjectPlugin
+    {
+    }
+
+    // probably seggregate this out.
+    public interface IPluginActions
+    {
+        IStartupWorkflow CreateWorkflow();
+        ILog GetLog();
+        ITimeSet GetTimeSet();
+    }
+}

DistTestCore/TestLifecycle.cs

@@ -26,6 +26,14 @@ namespace DistTestCore
             testStart = DateTime.UtcNow;
             //CodexVersion = null;
 
+            // the plugin manager is starting to look like the testlifecycle, that's bad because they are not supposed to be doing the same things:
+            // pluginmanager should be useful for disttest-deployer-continuoustest, everyone!
+            // but testlifecycle should be a disttest specific user of the plugin manager.
+            // disttest requires a hook by which it can keep track of containers created?? (does it?) /namespace used? for the purpose of cleaning up.
+
+            //var pluginManager = new PluginManager(Log, configuration, timeSet, testNamespace);
+            //pluginManager.InitializeAllPlugins();
+
             Log.WriteLogTag();
         }
 

KubernetesWorkflow/RunnerLocationUtils.cs

@@ -0,0 +1,55 @@
+using System.Net.NetworkInformation;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    internal enum RunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
+    }
+
+    internal static class RunnerLocationUtils
+    {
+        private static RunnerLocation? knownLocation = null;
+
+        internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            if (knownLocation != null) return knownLocation.Value;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                knownLocation = RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                knownLocation = RunnerLocation.ExternalToCluster;
+            }
+
+            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
+            return knownLocation.Value;
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
+}

KubernetesWorkflow/RunningContainers.cs

@@ -43,6 +43,19 @@ namespace KubernetesWorkflow
 
         [JsonIgnore]
         public CrashWatcher? CrashWatcher { get; set; }
+
+        [JsonIgnore]
+        public Address Address
+        {
+            get
+            {
+                if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
+                {
+                    return ClusterInternalAddress;
+                }
+                return ClusterExternalAddress;
+            }
+        }
     }
 
     public static class RunningContainersExtensions

KubernetesWorkflow/StartupConfig.cs

@@ -5,6 +5,7 @@
         private readonly List<object> configs = new List<object>();
 
         public string? NameOverride { get; set; }
+        public bool CreateCrashWatcher { get; set; }
 
         public void Add(object config)
         {

KubernetesWorkflow/StartupWorkflow.cs

@@ -3,7 +3,17 @@ using Utils;
 
 namespace KubernetesWorkflow
 {
-    public class StartupWorkflow
+    public interface IStartupWorkflow
+    {
+        RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        void Stop(RunningContainers runningContainers);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines);
+        string ExecuteCommand(RunningContainer container, string command, params string[] args);
+        void DeleteAllResources();// !!!  delete namespace then!?
+        void DeleteTestResources(); // !!! do not mention tests. what are we deleting?
+    }
+
+    public class StartupWorkflow : IStartupWorkflow
     {
         private readonly BaseLog log;
         private readonly WorkflowNumberSource numberSource;
@@ -26,18 +36,15 @@ namespace KubernetesWorkflow
             return K8s(controller =>
             {
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
-
                 var runningPod = controller.BringOnline(recipes, location);
+                var containers = CreateContainers(runningPod, recipes, startupConfig);
 
-                return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
+                if (startupConfig.CreateCrashWatcher) CreateCrashWatchers(controller, containers);
+
+                return new RunningContainers(startupConfig, runningPod, containers);
             });
         }
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
-        {
-            return K8s(controller => controller.CreateCrashWatcher(container));
-        }
-
         public void Stop(RunningContainers runningContainers)
         {
             K8s(controller =>
@@ -78,6 +85,14 @@ namespace KubernetesWorkflow
             });
         }
 
+        private void CreateCrashWatchers(K8sController controller, RunningContainer[] runningContainers)
+        {
+            foreach (var container in runningContainers)
+            {
+                container.CrashWatcher = controller.CreateCrashWatcher(container);
+            }
+        }
+
         private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
             log.Debug();

KubernetesWorkflow/WorkflowCreator.cs

@@ -19,7 +19,7 @@ namespace KubernetesWorkflow
             this.testNamespace = testNamespace.ToLowerInvariant();
         }
 
-        public StartupWorkflow CreateWorkflow()
+        public IStartupWorkflow CreateWorkflow()
         {
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);

Logging/BaseLog.cs

@@ -1,8 +1,17 @@
-using Utils;
+using System.Diagnostics;
+using Utils;
 
 namespace Logging
 {
-    public abstract class BaseLog
+    public interface ILog
+    {
+        void Log(string message);
+        void Debug(string message = "", int skipFrames = 0);
+        void Error(string message);
+        LogFile CreateSubfile(string ext = "log");
+    }
+
+    public abstract class BaseLog : ILog
     {
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly bool debug;

Logging/Stopwatch.cs

@@ -5,25 +5,25 @@ namespace Logging
     public class Stopwatch
     {
         private readonly DateTime start = DateTime.UtcNow;
-        private readonly BaseLog log;
+        private readonly ILog log;
         private readonly string name;
         private readonly bool debug;
 
-        private Stopwatch(BaseLog log, string name, bool debug)
+        private Stopwatch(ILog log, string name, bool debug)
         {
             this.log = log;
             this.name = name;
             this.debug = debug;
         }
 
-        public static void Measure(BaseLog log, string name, Action action, bool debug = false)
+        public static void Measure(ILog log, string name, Action action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             action();
             sw.End();
         }
 
-        public static T Measure<T>(BaseLog log, string name, Func<T> action, bool debug = false)
+        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             var result = action();
@@ -31,22 +31,22 @@ namespace Logging
             return result;
         }
 
-        public static Stopwatch Begin(BaseLog log)
+        public static Stopwatch Begin(ILog log)
         {
             return Begin(log, "");
         }
 
-        public static Stopwatch Begin(BaseLog log, string name)
+        public static Stopwatch Begin(ILog log, string name)
         {
             return Begin(log, name, false);
         }
 
-        public static Stopwatch Begin(BaseLog log, bool debug)
+        public static Stopwatch Begin(ILog log, bool debug)
         {
             return Begin(log, "", debug);
         }
 
-        public static Stopwatch Begin(BaseLog log, string name, bool debug)
+        public static Stopwatch Begin(ILog log, string name, bool debug)
         {
             return new Stopwatch(log, name, debug);
         }

Tests/Tests.csproj

@@ -14,7 +14,6 @@
 
   <ItemGroup>
     <ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>