Merge branch 'feature/containerless-codex'

# Conflicts:
#	Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

.gitignore

@@ -3,3 +3,4 @@ obj
 bin
 .vscode
 Tools/AutoClient/datapath
+.editorconfig

Framework/Core/Core.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
     <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\WebUtils\WebUtils.csproj" />
   </ItemGroup>
 
 </Project>

Framework/Core/CoreInterface.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;
 
 namespace Core
 {

Framework/Core/EntryPoint.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -8,16 +9,16 @@ namespace Core
         private readonly IToolsFactory toolsFactory;
         private readonly PluginManager manager = new PluginManager();
 
-        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
-            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
 
             Tools = toolsFactory.CreateTools();
             manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
         }
 
         public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
-            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+            : this(log, configuration, fileManagerRootFolder, new DefaultWebCallTimeSet(), new DefaultK8sTimeSet())
         {
         }
 

Framework/Core/PluginTools.cs

@@ -1,12 +1,14 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
-    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
     {
-        ITimeSet TimeSet { get; }
+        IWebCallTimeSet WebCallTimeSet { get; }
+        IK8sTimeSet K8STimeSet { get; }
 
         /// <summary>
         /// Deletes kubernetes and tracked file resources.
@@ -25,13 +27,6 @@ namespace Core
         ILog GetLog();
     }
 
-    public interface IHttpFactoryTool
-    {
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp(string id);
-    }
-
     public interface IFileTool
     {
         IFileManager GetFileManager();
@@ -40,18 +35,22 @@ namespace Core
     internal class PluginTools : IPluginTools
     {
         private readonly WorkflowCreator workflowCreator;
+        private readonly HttpFactory httpFactory;
         private readonly IFileManager fileManager;
         private readonly LogPrefixer log;
 
-        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            TimeSet = timeSet;
+            httpFactory = new HttpFactory(log, webCallTimeSet);
+            WebCallTimeSet = webCallTimeSet;
+            K8STimeSet = k8STimeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
-        public ITimeSet TimeSet { get; }
+        public IWebCallTimeSet WebCallTimeSet { get; }
+        public IK8sTimeSet K8STimeSet { get; }
 
         public void ApplyLogPrefix(string prefix)
         {
@@ -60,17 +59,17 @@ namespace Core
 
         public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(id, onClientCreated, TimeSet);
+            return httpFactory.CreateHttp(id, onClientCreated);
         }
 
-        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet)
         {
-            return new Http(id, log, ts, onClientCreated);
+            return httpFactory.CreateHttp(id, onClientCreated, timeSet);
         }
 
         public IHttp CreateHttp(string id)
         {
-            return new Http(id, log, TimeSet);
+            return httpFactory.CreateHttp(id);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/Core/ToolsFactory.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -13,19 +14,21 @@ namespace Core
         private readonly ILog log;
         private readonly WorkflowCreator workflowCreator;
         private readonly string fileManagerRootFolder;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet webCallTimeSet;
+        private readonly IK8sTimeSet k8STimeSet;
 
-        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = log;
             workflowCreator = new WorkflowCreator(log, configuration);
             this.fileManagerRootFolder = fileManagerRootFolder;
-            this.timeSet = timeSet;
+            this.webCallTimeSet = webCallTimeSet;
+            this.k8STimeSet = k8STimeSet;
         }
 
         public PluginTools CreateTools()
         {
-            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
         }
     }
 }

Framework/FileUtils/TrackedFile.cs

@@ -14,6 +14,12 @@ namespace FileUtils
             Label = label;
         }
 
+        public static TrackedFile FromPath(ILog log, string filepath)
+        {
+            // todo: I don't wanne have to do this to call upload.
+            return new TrackedFile(log, filepath, string.Empty);
+        }
+
         public string Filename { get; }
         public string Label { get; }
 

Framework/KubernetesWorkflow/ContainerCrashWatcher.cs

@@ -3,7 +3,7 @@ using Logging;
 
 namespace KubernetesWorkflow
 {
-    public class CrashWatcher
+    public class ContainerCrashWatcher
     {
         private readonly ILog log;
         private readonly KubernetesClientConfiguration config;
@@ -15,7 +15,7 @@ namespace KubernetesWorkflow
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
@@ -45,7 +45,7 @@ namespace KubernetesWorkflow
             if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
         }
 
-        public bool HasContainerCrashed()
+        public bool HasCrashed()
         {
             using var client = new Kubernetes(config);
             var result = HasContainerBeenRestarted(client);

Framework/KubernetesWorkflow/K8sController.cs

@@ -946,13 +946,13 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             var containerName = container.Name;
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new ContainerCrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/K8sTimeSet.cs

@@ -0,0 +1,42 @@
+namespace Core
+{
+    public interface IK8sTimeSet
+    {
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}

Framework/KubernetesWorkflow/LogHandler.cs

@@ -29,7 +29,7 @@ namespace KubernetesWorkflow
         {
             LogFile = sourceLog.CreateSubfile(addFileName);
 
-            var msg = $"{description} -->> {LogFile.FullFilename}";
+            var msg = $"{description} -->> {LogFile.Filename}";
             sourceLog.Log(msg);
 
             LogFile.Write(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
         FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
         PodInfo GetPodInfo(RunningPod pod);
-        CrashWatcher CreateCrashWatcher(RunningContainer container);
+        ContainerCrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
         IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
@@ -93,7 +93,7 @@ namespace KubernetesWorkflow
             return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             return K8s(c => c.CreateCrashWatcher(container));
         }
@@ -134,7 +134,7 @@ namespace KubernetesWorkflow
                 controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
 
-            return new DownloadedLog(logHandler, container.Name);
+            return new DownloadedLog(logHandler.LogFile, container.Name);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -209,6 +209,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetExternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{recipe.Name}:{tag}",
                 startResult.Cluster.HostAddress,
                 port.Number);
         }
@@ -220,6 +221,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetInternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{serviceName}:{tag}",
                 $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                 port.Number);
         }

Framework/Logging/BaseLog.cs

@@ -34,7 +34,7 @@ namespace Logging
         {
             get
             {
-                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
+                if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
                 return logFile;
             }
         }
@@ -69,7 +69,7 @@ namespace Logging
 
         public virtual void Delete()
         {
-            File.Delete(LogFile.FullFilename);
+            File.Delete(LogFile.Filename);
         }
 
         public LogFile CreateSubfile(string addName, string ext = "log")
@@ -78,7 +78,7 @@ namespace Logging
                 .Replace("<", "")
                 .Replace(">", "");
 
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/DownloadedLog.cs

@@ -1,10 +1,8 @@
-using Logging;
-
-namespace KubernetesWorkflow
+namespace Logging
 {
     public interface IDownloadedLog
     {
-        string ContainerName { get; }
+        string SourceName { get; }
 
         void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
@@ -14,21 +12,27 @@ namespace KubernetesWorkflow
         void DeleteFile();
     }
 
-    internal class DownloadedLog : IDownloadedLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
+        public DownloadedLog(string filepath, string sourceName)
         {
-            logFile = logHandler.LogFile;
-            ContainerName = containerName;
+            logFile = new LogFile(filepath);
+            SourceName = sourceName;
         }
 
-        public string ContainerName { get; }
+        public DownloadedLog(LogFile logFile, string sourceName)
+        {
+            this.logFile = logFile;
+            SourceName = sourceName;
+        }
+
+        public string SourceName { get; }
 
         public void IterateLines(Action<string> action)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
+            using var file = File.OpenRead(logFile.Filename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
@@ -64,12 +68,12 @@ namespace KubernetesWorkflow
 
         public string GetFilepath()
         {
-            return logFile.FullFilename;
+            return logFile.Filename;
         }
 
         public void DeleteFile()
         {
-            File.Delete(logFile.FullFilename);
+            File.Delete(logFile.Filename);
         }
     }
 }

Framework/Logging/LogFile.cs

@@ -4,20 +4,16 @@ namespace Logging
 {
     public class LogFile
     {
-        private readonly string extension;
         private readonly object fileLock = new object();
-        private string filename;
 
-        public LogFile(string filename, string extension)
+        public LogFile(string filename)
         {
-            this.filename = filename;
-            this.extension = extension;
-            FullFilename = filename + "." + extension;
+            Filename = filename;
 
             EnsurePathExists(filename);
         }
 
-        public string FullFilename { get; private set; }
+        public string Filename { get; private set; }
 
         public void Write(string message)
         {
@@ -30,7 +26,7 @@ namespace Logging
             {
                 lock (fileLock)
                 { 
-                    File.AppendAllLines(FullFilename, new[] { message });
+                    File.AppendAllLines(Filename, new[] { message });
                 }
             }
             catch (Exception ex)
@@ -39,14 +35,19 @@ namespace Logging
             }
         }
 
-        public void ConcatToFilename(string toAdd)
+        public void WriteRawMany(IEnumerable<string> lines)
         {
-            var oldFullName = FullFilename;
-
-            filename += toAdd;
-            FullFilename = filename + "." + extension;
-
-            File.Move(oldFullName, FullFilename);
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(Filename, lines);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
         }
 
         private static string GetTimestamp()

Framework/Utils/Address.cs

@@ -2,12 +2,14 @@
 {
     public class Address
     {
-        public Address(string host, int port)
+        public Address(string logName, string host, int port)
         {
+            LogName = logName;
             Host = host;
             Port = port;
         }
 
+        public string LogName { get; }
         public string Host { get; }
         public int Port { get; }
 
@@ -20,5 +22,21 @@
         {
             return !string.IsNullOrEmpty(Host) && Port > 0;
         }
+
+        public static Address Empty()
+        {
+            return new Address(string.Empty, string.Empty, 0);
+        }
     }
+
+    public interface IHasMetricsScrapeTarget
+    {
+        Address GetMetricsScrapeTarget();
+    }
+    
+    public interface IHasManyMetricScrapeTargets
+    {
+        Address[] GetMetricsScrapeTargets();
+    }
+
 }

Framework/Utils/CrashWatcher.cs

@@ -0,0 +1,9 @@
+namespace Utils
+{
+    //public interface ICrashWatcher
+    //{
+    //    void Start();
+    //    void Stop();
+    //    bool HasCrashed();
+    //}
+}

Framework/Utils/EthAccount.cs

@@ -0,0 +1,20 @@
+namespace Utils
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
+    }
+}

Framework/Utils/EthAddress.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public interface IHasEthAddress
     {

Framework/Utils/EthTokenExtensions.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public class Ether : IComparable<Ether>
     {

Framework/Utils/TestTokenExtensions.cs

@@ -1,6 +1,6 @@
 using System.Numerics;
 
-namespace CodexContractsPlugin
+namespace Utils
 {
     public class TestToken : IComparable<TestToken>
     {

Framework/WebUtils/Endpoint.cs

@@ -1,11 +1,10 @@
-using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using Logging;
+using Newtonsoft.Json;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IEndpoint
     {
@@ -119,7 +118,7 @@ namespace Core
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
             {
-                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                Error = delegate (object? sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
                 {
                     if (args.CurrentObject == args.ErrorContext.OriginalObject)
                     {

Framework/WebUtils/Http.cs

@@ -1,7 +1,7 @@
 using Logging;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IHttp
     {
@@ -16,16 +16,16 @@ namespace Core
         private static object lockLock = new object();
         private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet timeSet;
         private readonly Action<HttpClient> onClientCreated;
         private readonly string id;
 
-        internal Http(string id, ILog log, ITimeSet timeSet)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet)
             : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.id = id;
             this.log = log;

Framework/WebUtils/HttpFactory.cs

@@ -0,0 +1,43 @@
+using Logging;
+
+namespace WebUtils
+{
+    public interface IHttpFactory
+    {
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
+        IHttp CreateHttp(string id);
+    }
+
+    public class HttpFactory : IHttpFactory
+    {
+        private readonly ILog log;
+        private readonly IWebCallTimeSet defaultTimeSet;
+
+        public HttpFactory(ILog log)
+            : this (log, new DefaultWebCallTimeSet())
+        {
+        }
+
+        public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet)
+        {
+            this.log = log;
+            this.defaultTimeSet = defaultTimeSet;
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(id, onClientCreated, defaultTimeSet);
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
+        {
+            return new Http(id, log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp(string id)
+        {
+            return new Http(id, log, defaultTimeSet);
+        }
+    }
+}

Framework/WebUtils/WebCallTimeSet.cs

@@ -1,6 +1,6 @@
-namespace Core
+namespace WebUtils
 {
-    public interface ITimeSet
+    public interface IWebCallTimeSet
     {
         /// <summary>
         /// Timeout for a single HTTP call.
@@ -17,20 +17,9 @@
         /// After a failed HTTP call, wait this long before trying again.
         /// </summary>
         TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan K8sOperationTimeout();
     }
 
-    public class DefaultTimeSet : ITimeSet
+    public class DefaultWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -46,19 +35,9 @@
         {
             return TimeSpan.FromSeconds(1);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(30);
-        }
     }
 
-    public class LongTimeSet : ITimeSet
+    public class LongWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -74,15 +53,5 @@
         {
             return TimeSpan.FromSeconds(20);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromHours(1);
-        }
     }
 }

Framework/WebUtils/WebUtils.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	</ItemGroup>
+
+	<ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -1,31 +1,50 @@
 using CodexOpenApi;
-using Core;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
+using WebUtils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class CodexAccess
     {
         private readonly ILog log;
-        private readonly IPluginTools tools;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControl processControl;
+        private ICodexInstance instance;
         private readonly Mapper mapper = new Mapper();
 
-        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
+        public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
         {
-            this.tools = tools;
-            log = tools.GetLog();
-            Container = container;
-            CrashWatcher = crashWatcher;
-
-            CrashWatcher.Start();
+            this.log = log;
+            this.httpFactory = httpFactory;
+            this.processControl = processControl;
+            this.instance = instance;
         }
 
-        public RunningPod Container { get; }
-        public CrashWatcher CrashWatcher { get; }
+        public void Stop(bool waitTillStopped)
+        {
+            processControl.Stop(waitTillStopped);
+            // Prevents accidental use after stop:
+            instance = null!;
+        }
+
+        public IDownloadedLog DownloadLog(string additionalName = "")
+        {
+            var file = log.CreateSubfile(GetName() + additionalName);
+            Log($"Downloading logs to '{file.Filename}'");
+            return processControl.DownloadLog(file);
+        }
+
+        public string GetImageName()
+        {
+            return instance.ImageName;
+        }
+
+        public DateTime GetStartUtc()
+        {
+            return instance.StartUtc;
+        }
 
         public DebugInfo GetDebugInfo()
         {
@@ -79,19 +98,14 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput)
         {
-            return OnCodex(
-                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
-                CreateRetryConfig(nameof(UploadFile), onFailure));
+            return OnCodex(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream));
         }
 
-        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
+        public Stream DownloadFile(string contentId)
         {
-            var fileResponse = OnCodex(
-                api => api.DownloadNetworkStreamAsync(contentId),
-                CreateRetryConfig(nameof(DownloadFile), onFailure));
-
+            var fileResponse = OnCodex(api => api.DownloadNetworkStreamAsync(contentId));
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
@@ -123,7 +137,6 @@ namespace CodexPlugin
                     return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
                 }, nameof(LocalFiles));
             }));
-
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
@@ -170,47 +183,60 @@ namespace CodexPlugin
 
         public string GetName()
         {
-            return Container.Name;
+            return instance.Name;
         }
 
-        public PodInfo GetPodInfo()
+        public Address GetDiscoveryEndpoint()
         {
-            var workflow = tools.CreateWorkflow();
-            return workflow.GetPodInfo(Container);
+            return instance.DiscoveryEndpoint;
         }
 
-        public void DeleteRepoFolder()
+        public Address GetApiEndpoint()
         {
-            try
-            {
-                var containerNumber = Container.Containers.First().Recipe.Number;
-                var dataDir = $"datadir{containerNumber}";
-                var workflow = tools.CreateWorkflow();
-                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
-                Log("Deleted repo folder.");
-            }
-            catch (Exception e)
-            {
-                Log("Unable to delete repo folder: " + e);
-            }
+            return instance.ApiEndpoint;
         }
 
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
+        public Address GetListenEndpoint()
         {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
+            return instance.ListenEndpoint;
+        }
+
+        public bool HasCrashed()
+        {
+            return processControl.HasCrashed();
+        }
+
+        public Address? GetMetricsEndpoint()
+        {
+            return instance.MetricsEndpoint;
+        }
+
+        public EthAccount? GetEthAccount()
+        {
+            return instance.EthAccount;
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            processControl.DeleteDataDirFolder();
+        }
+
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
+        {
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
             return result;
         }
 
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action, Retry retry)
         {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), retry);
             return result;
         }
 
-        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
+        private T CallCodex<T>(HttpClient client, Func<CodexApiClient, Task<T>> action)
         {
             var address = GetAddress();
-            var api = new CodexApi(client);
+            var api = new CodexApiClient(client);
             api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
             return CrashCheck(() => Time.Wait(action(api)));
         }
@@ -223,20 +249,20 @@ namespace CodexPlugin
             }
             finally
             {
-                CrashWatcher.HasContainerCrashed();
+                CheckContainerCrashed();
             }
         }
 
         private IEndpoint GetEndpoint()
         {
-            return tools
-                .CreateHttp(GetHttpId(), CheckContainerCrashed)
-                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
+            return httpFactory
+                .CreateHttp(GetHttpId(), h => CheckContainerCrashed())
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", GetName());
         }
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
+            return instance.ApiEndpoint;
         }
 
         private string GetHttpId()
@@ -244,52 +270,9 @@ namespace CodexPlugin
             return GetAddress().ToString();
         }
 
-        private void CheckContainerCrashed(HttpClient client)
+        private void CheckContainerCrashed()
         {
-            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
-        }
-
-        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
-        {
-            var timeSet = tools.TimeSet;
-
-            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
-            {
-                onFailure(failure);
-                Investigate(failure, timeSet);
-            });
-        }
-
-        private void Investigate(Failure failure, ITimeSet timeSet)
-        {
-            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
-                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
-                $"Checking if node responds to debug/info...");
-
-            try
-            {
-                var debugInfo = GetDebugInfo();
-                if (string.IsNullOrEmpty(debugInfo.Spr))
-                {
-                    Log("Did not get value debug/info response.");
-                    Throw(failure);
-                }
-                else
-                {
-                    Log("Got valid response from debug/info.");
-                }
-            }
-            catch (Exception ex)
-            {
-                Log("Got exception from debug/info call: " + ex);
-                Throw(failure);
-            }
-
-            if (failure.Duration < timeSet.HttpCallTimeout())
-            {
-                Log("Retry failed within HTTP timeout duration.");
-                Throw(failure);
-            }
+            if (processControl.HasCrashed()) throw new Exception($"Container {GetName()} has crashed.");
         }
 
         private void Throw(Failure failure)
@@ -299,7 +282,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{GetName()} {msg}");
+            log.Log($"({GetName()}) {msg}");
         }
     }
 

ProjectPlugins/CodexClient/CodexClient.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApiClient" />
+  </ItemGroup>
+
+<ItemGroup>
+	<PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+	<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	<PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+</ItemGroup>
+	
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\Framework\WebUtils\WebUtils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexInstance.cs

@@ -0,0 +1,54 @@
+using Utils;
+
+namespace CodexClient
+{
+    public interface ICodexInstance
+    {
+        string Name { get; }
+        string ImageName { get; }
+        DateTime StartUtc { get; }
+        Address DiscoveryEndpoint { get; }
+        Address ApiEndpoint { get; }
+        Address ListenEndpoint { get; }
+        EthAccount? EthAccount { get; }
+        Address? MetricsEndpoint { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint)
+        {
+            Name = name;
+            ImageName = imageName;
+            StartUtc = startUtc;
+            DiscoveryEndpoint = discoveryEndpoint;
+            ApiEndpoint = apiEndpoint;
+            ListenEndpoint = listenEndpoint;
+            EthAccount = ethAccount;
+            MetricsEndpoint = metricsEndpoint;
+        }
+
+        public string Name { get; }
+        public string ImageName { get; }
+        public DateTime StartUtc { get; }
+        public Address DiscoveryEndpoint { get; }
+        public Address ApiEndpoint { get; }
+        public Address ListenEndpoint { get; }
+        public EthAccount? EthAccount { get; }
+        public Address? MetricsEndpoint { get; }
+
+        public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint)
+        {
+            return new CodexInstance(
+                name,
+                imageName: "-",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: Address.Empty(),
+                apiEndpoint: apiEndpoint,
+                listenEndpoint: Address.Empty(),
+                ethAccount: null,
+                metricsEndpoint: null
+            );
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexLogLevel.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public enum CodexLogLevel
     {

ProjectPlugins/CodexClient/CodexLogLine.cs

@@ -1,6 +1,6 @@
 using System.Globalization;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class CodexLogLine
     {

ProjectPlugins/CodexClient/CodexNode.cs

@@ -1,27 +1,21 @@
-using CodexPlugin.Hooks;
-using Core;
+using CodexClient.Hooks;
 using FileUtils;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
 using Logging;
-using MetricsPlugin;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
-    public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
+    public partial interface ICodexNode : IHasEthAddress, IHasMetricsScrapeTarget
     {
         string GetName();
+        string GetImageName();
         string GetPeerId();
         DebugInfo GetDebugInfo(bool log = false);
         string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
-        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
-        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
         LocalDataset DownloadStreamless(ContentId cid);
         /// <summary>
         /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
@@ -34,70 +28,67 @@ namespace CodexPlugin
         void ConnectToPeer(ICodexNode node);
         DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
-        CrashWatcher CrashWatcher { get; }
-        PodInfo GetPodInfo();
         ITransferSpeeds TransferSpeeds { get; }
         EthAccount EthAccount { get; }
+        StoragePurchase GetPurchaseStatus(string purchaseId);
+
+        Address GetDiscoveryEndpoint();
+        Address GetApiEndpoint();
+        Address GetListenEndpoint();
 
         /// <summary>
         /// Warning! The node is not usable after this.
         /// TODO: Replace with delete-blocks debug call once available in Codex.
         /// </summary>
-        void DeleteRepoFolder();
+        void DeleteDataDirFolder();
         void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(string additionalName = "");
+        bool HasCrashed();
     }
 
     public class CodexNode : ICodexNode
     {
         private const string UploadFailedMessage = "Unable to store block";
         private readonly ILog log;
-        private readonly IPluginTools tools;
         private readonly ICodexNodeHooks hooks;
-        private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
         private string peerId = string.Empty;
         private string nodeId = string.Empty;
+        private readonly CodexAccess codexAccess;
+        private readonly IFileManager fileManager;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
+        public CodexNode(ILog log, CodexAccess codexAccess, IFileManager fileManager, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks)
         {
-            this.tools = tools;
-            this.ethAccount = ethAccount;
-            CodexAccess = codexAccess;
-            Group = group;
+            this.codexAccess = codexAccess;
+            this.fileManager = fileManager;
             Marketplace = marketplaceAccess;
             this.hooks = hooks;
             Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
 
-            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
+            this.log = new LogPrefixer(log, $"{GetName()} ");
         }
 
         public void Awake()
         {
-            hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image, ethAccount);
+            hooks.OnNodeStarting(codexAccess.GetStartUtc(), codexAccess.GetImageName(), codexAccess.GetEthAccount());
         }
 
         public void Initialize()
         {
-            hooks.OnNodeStarted(peerId, nodeId);
+            InitializePeerNodeId();
+            InitializeLogReplacements();
+
+            hooks.OnNodeStarted(this, peerId, nodeId);
         }
 
-        public RunningPod Pod { get { return CodexAccess.Container; } }
-        
-        public RunningContainer Container { get { return Pod.Containers.Single(); } }
-        public CodexAccess CodexAccess { get; }
-        public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
-        public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
         public DebugInfoVersion Version { get; private set; }
         public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
 
-        public IMetricsScrapeTarget MetricsScrapeTarget
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            get
-            {
-                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
-            }
+            return codexAccess.GetPurchaseStatus(purchaseId);
         }
 
         public EthAddress EthAddress 
@@ -105,7 +96,7 @@ namespace CodexPlugin
             get
             {
                 EnsureMarketplace();
-                return ethAccount!.EthAddress;
+                return codexAccess.GetEthAccount()!.EthAddress;
             }
         }
 
@@ -114,13 +105,18 @@ namespace CodexPlugin
             get
             {
                 EnsureMarketplace();
-                return ethAccount!;
+                return codexAccess.GetEthAccount()!;
             }
         }
 
         public string GetName()
         {
-            return Container.Name;
+            return codexAccess.GetName();
+        }
+
+        public string GetImageName()
+        {
+            return codexAccess.GetImageName();
         }
 
         public string GetPeerId()
@@ -130,7 +126,7 @@ namespace CodexPlugin
 
         public DebugInfo GetDebugInfo(bool log = false)
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = codexAccess.GetDebugInfo();
             if (log)
             {
                 var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
@@ -141,25 +137,20 @@ namespace CodexPlugin
 
         public string GetSpr()
         {
-            return CodexAccess.GetSpr();
+            return codexAccess.GetSpr();
         }
 
         public DebugPeer GetDebugPeer(string peerId)
         {
-            return CodexAccess.GetDebugPeer(peerId);
+            return codexAccess.GetDebugPeer(peerId);
         }
 
         public ContentId UploadFile(TrackedFile file)
         {
-            return UploadFile(file, DoNothing);
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"");
         }
 
-        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
-        {
-            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
-        }
-
-        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -171,7 +162,7 @@ namespace CodexPlugin
             var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(input, onFailure);
+                return codexAccess.UploadFile(input);
             });
 
             var response = measurement.Value;
@@ -189,17 +180,12 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
-        }
-
-        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
-        {
-            var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
+            var file = fileManager.CreateEmptyFile(fileLabel);
             hooks.OnFileDownloading(contentId);
             Log($"Downloading '{contentId}'...");
 
             var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
-            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file));
 
             var size = file.GetFilesize();
             transferSpeeds.AddDownloadSample(size, measurement);
@@ -211,7 +197,7 @@ namespace CodexPlugin
         public LocalDataset DownloadStreamless(ContentId cid)
         {
             Log($"Downloading streamless '{cid}' (no-wait)");
-            return CodexAccess.DownloadStreamless(cid);
+            return codexAccess.DownloadStreamless(cid);
         }
 
         public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
@@ -221,7 +207,7 @@ namespace CodexPlugin
             var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
             {
                 var startSpace = Space();
-                var result = CodexAccess.DownloadStreamless(cid);
+                var result = codexAccess.DownloadStreamless(cid);
                 WaitUntilQuotaUsedIncreased(startSpace, size);
                 return result;
             });
@@ -232,17 +218,17 @@ namespace CodexPlugin
         public LocalDataset DownloadManifestOnly(ContentId cid)
         {
             Log($"Downloading manifest-only '{cid}'");
-            return CodexAccess.DownloadManifestOnly(cid);
+            return codexAccess.DownloadManifestOnly(cid);
         }
 
         public LocalDatasetList LocalFiles()
         {
-            return CodexAccess.LocalFiles();
+            return codexAccess.LocalFiles();
         }
 
         public CodexSpace Space()
         {
-            return CodexAccess.Space();
+            return codexAccess.Space();
         }
 
         public void ConnectToPeer(ICodexNode node)
@@ -251,47 +237,53 @@ namespace CodexPlugin
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
+            codexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
 
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public PodInfo GetPodInfo()
+        public void DeleteDataDirFolder()
         {
-            return CodexAccess.GetPodInfo();
-        }
-
-        public void DeleteRepoFolder()
-        {
-            CodexAccess.DeleteRepoFolder();
+            codexAccess.DeleteDataDirFolder();
         }
 
         public void Stop(bool waitTillStopped)
         {
             Log("Stopping...");
             hooks.OnNodeStopping();
-
-            CrashWatcher.Stop();
-            Group.Stop(this, waitTillStopped);
+            codexAccess.Stop(waitTillStopped);
         }
 
-        public void EnsureOnlineGetVersionResponse()
+        public IDownloadedLog DownloadLog(string additionalName = "")
         {
-            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            peerId = debugInfo.Id;
-            nodeId = debugInfo.Table.LocalNode.NodeId;
-            var nodeName = CodexAccess.Container.Name;
+            return codexAccess.DownloadLog(additionalName);
+        }
 
-            if (!debugInfo.Version.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
-            }
+        public Address GetDiscoveryEndpoint()
+        {
+            return codexAccess.GetDiscoveryEndpoint();
+        }
 
-            log.AddStringReplace(peerId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
-            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(debugInfo.Table.LocalNode.NodeId), nodeName);
-            Version = debugInfo.Version;
+        public Address GetApiEndpoint()
+        {
+            return codexAccess.GetApiEndpoint();
+        }
+
+        public Address GetListenEndpoint()
+        {
+            return codexAccess.GetListenEndpoint();
+        }
+
+        public Address GetMetricsScrapeTarget()
+        {
+            var address = codexAccess.GetMetricsEndpoint();
+            if (address == null) throw new Exception("Metrics ScrapeTarget accessed, but node was not started with EnableMetrics()");
+            return address;
+        }
+
+        public bool HasCrashed()
+        {
+            return codexAccess.HasCrashed();
         }
 
         public override string ToString()
@@ -299,22 +291,44 @@ namespace CodexPlugin
             return $"CodexNode:{GetName()}";
         }
 
+        private void InitializePeerNodeId()
+        {
+            var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online");
+            if (!debugInfo.Version.IsValid())
+            {
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
+            }
+
+            peerId = debugInfo.Id;
+            nodeId = debugInfo.Table.LocalNode.NodeId;
+            Version = debugInfo.Version;
+        }
+
+        private void InitializeLogReplacements()
+        {
+            var nodeName = GetName();
+
+            log.AddStringReplace(peerId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
+            log.AddStringReplace(nodeId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName);
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Pod);
+            var peerId = peer.GetDiscoveryEndpoint().Host
+                .Replace("http://", "")
+                .Replace("https://", "");
 
             return peerInfo.Addrs.Select(a => a
-                .Replace("0.0.0.0", podInfo.Ip))
+                .Replace("0.0.0.0", peerId))
                 .ToArray();
         }
 
-        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
+        private void DownloadToFile(string contentId, TrackedFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);
-            var timeout = tools.TimeSet.HttpCallTimeout();
+            var timeout = TimeSpan.FromMinutes(2.0); // todo: make this user-controllable.
             try
             {
                 // Type of stream generated by openAPI client does not support timeouts.
@@ -322,7 +336,7 @@ namespace CodexPlugin
                 var cts = new CancellationTokenSource();
                 var downloadTask = Task.Run(() =>
                 {
-                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    using var downloadStream = codexAccess.DownloadFile(contentId);
                     downloadStream.CopyTo(fileStream);
                 }, cts.Token);
                 
@@ -336,9 +350,9 @@ namespace CodexPlugin
                 cts.Cancel();
                 throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
-            catch
+            catch (Exception ex)
             {
-                Log($"Failed to download file '{contentId}'.");
+                Log($"Failed to download file '{contentId}': {ex}");
                 throw;
             }
         }
@@ -378,16 +392,12 @@ namespace CodexPlugin
 
         private void EnsureMarketplace()
         {
-            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
+            if (codexAccess.GetEthAccount() == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
         }
 
         private void Log(string msg)
         {
             log.Log(msg);
         }
-
-        private void DoNothing(Failure failure)
-        {
-        }
     }
 }

ProjectPlugins/CodexClient/CodexNodeFactory.cs

@@ -0,0 +1,47 @@
+using CodexClient.Hooks;
+using FileUtils;
+using Logging;
+using WebUtils;
+
+namespace CodexClient
+{
+    public class CodexNodeFactory
+    {
+        private readonly ILog log;
+        private readonly IFileManager fileManager;
+        private readonly CodexHooksFactory hooksFactory;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControlFactory processControlFactory;
+
+        public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory)
+        {
+            this.log = log;
+            this.fileManager = fileManager;
+            this.hooksFactory = hooksFactory;
+            this.httpFactory = httpFactory;
+            this.processControlFactory = processControlFactory;
+        }
+
+        public CodexNodeFactory(ILog log, string dataDir)
+            : this(log, new FileManager(log, dataDir), new CodexHooksFactory(), new HttpFactory(log), new DoNothingProcessControlFactory())
+        {
+        }
+
+        public ICodexNode CreateCodexNode(ICodexInstance instance)
+        {
+            var processControl = processControlFactory.CreateProcessControl(instance);
+            var access = new CodexAccess(log, httpFactory, processControl, instance);
+            var hooks = hooksFactory.CreateHooks(access.GetName());
+            var marketplaceAccess = CreateMarketplaceAccess(instance, access, hooks);
+            var node =  new CodexNode(log, access, fileManager, marketplaceAccess, hooks);
+            node.Initialize();
+            return node;
+        }
+
+        private IMarketplaceAccess CreateMarketplaceAccess(ICodexInstance instance, CodexAccess access, ICodexNodeHooks hooks)
+        {
+            if (instance.EthAccount == null) return new MarketplaceUnavailable();
+            return new MarketplaceAccess(log, access, hooks);
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexTypes.cs

@@ -1,7 +1,7 @@
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class DebugInfo
     {

ProjectPlugins/CodexClient/CodexUtils.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public static class CodexUtils
     {

ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs

@@ -1,7 +1,6 @@
-using GethPlugin;
-using Utils;
+using Utils;
 
-namespace CodexPlugin.Hooks
+namespace CodexClient.Hooks
 {
     public interface ICodexHooksProvider
     {
@@ -10,11 +9,14 @@ namespace CodexPlugin.Hooks
 
     public class CodexHooksFactory
     {
-        public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
+        public List<ICodexHooksProvider> Providers { get; } = new List<ICodexHooksProvider>();
 
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
-            return Provider.CreateHooks(nodeName);
+            if (Providers.Count == 0) return new DoNothingCodexHooks();
+
+            var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray();
+            return new MuxingCodexNodeHooks(hooks);
         }
     }
 
@@ -44,7 +46,7 @@ namespace CodexPlugin.Hooks
         {
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
         }
 

ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs

@@ -0,0 +1,78 @@
+using Utils;
+
+namespace CodexClient.Hooks
+{
+    public interface ICodexNodeHooks
+    {
+        void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
+        void OnNodeStarted(ICodexNode node, string peerId, string nodeId);
+        void OnNodeStopping();
+        void OnFileUploading(string uid, ByteSize size);
+        void OnFileUploaded(string uid, ByteSize size, ContentId cid);
+        void OnFileDownloading(ContentId cid);
+        void OnFileDownloaded(ByteSize size, ContentId cid);
+        void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
+        void OnStorageContractUpdated(StoragePurchase purchaseStatus);
+        void OnStorageAvailabilityCreated(StorageAvailability response);
+    }
+
+    public class MuxingCodexNodeHooks : ICodexNodeHooks
+    {
+        private readonly ICodexNodeHooks[] backingHooks;
+
+        public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks)
+        {
+            this.backingHooks = backingHooks;
+        }
+
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloaded(size, cid);
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloading(cid);
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid);
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+            foreach (var h in backingHooks) h.OnFileUploading(uid, size);
+        }
+
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId);
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount);
+        }
+
+        public void OnNodeStopping()
+        {
+            foreach (var h in backingHooks) h.OnNodeStopping();
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+            foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response);
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract);
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus);
+        }
+    }
+}

ProjectPlugins/CodexClient/Mapper.cs

@@ -1,10 +1,8 @@
-using CodexContractsPlugin;
-using CodexOpenApi;
-using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Linq;
 using System.Numerics;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class Mapper
     {
@@ -80,12 +78,12 @@ namespace CodexPlugin
             };
         }
 
-        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        public StorageAvailability[] Map(ICollection<CodexOpenApi.SalesAvailabilityREAD> availabilities)
         {
             return availabilities.Select(a => Map(a)).ToArray();
         }
 
-        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD availability)
         {
             return new StorageAvailability
             (
@@ -142,7 +140,7 @@ namespace CodexPlugin
         //    };
         //}
 
-        public CodexSpace Map(Space space)
+        public CodexSpace Map(CodexOpenApi.Space space)
         {
             return new CodexSpace
             {
@@ -153,7 +151,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion Map(CodexVersion obj)
+        private DebugInfoVersion Map(CodexOpenApi.CodexVersion obj)
         {
             return new DebugInfoVersion
             {
@@ -162,7 +160,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTable Map(PeersTable obj)
+        private DebugInfoTable Map(CodexOpenApi.PeersTable obj)
         {
             return new DebugInfoTable
             {
@@ -171,7 +169,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode Map(Node? token)
+        private DebugInfoTableNode Map(CodexOpenApi.Node? token)
         {
             if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
@@ -184,7 +182,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
+        private DebugInfoTableNode[] Map(ICollection<CodexOpenApi.Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {

ProjectPlugins/CodexClient/MarketplaceAccess.cs

@@ -1,8 +1,8 @@
-using CodexPlugin.Hooks;
+using CodexClient.Hooks;
 using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IMarketplaceAccess
     {
@@ -72,7 +72,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
+            log.Log($"{codexAccess.GetName()} {msg}");
         }
     }
 

ProjectPlugins/CodexClient/MarketplaceTypes.cs

@@ -1,8 +1,7 @@
-using CodexContractsPlugin;
-using Logging;
+using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class StoragePurchaseRequest
     {

ProjectPlugins/CodexClient/ProcessControl.cs

@@ -0,0 +1,46 @@
+using Logging;
+
+namespace CodexClient
+{
+    public interface IProcessControlFactory
+    {
+        IProcessControl CreateProcessControl(ICodexInstance instance);
+    }
+
+    public interface IProcessControl
+    {
+        void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(LogFile file);
+        void DeleteDataDirFolder();
+        bool HasCrashed();
+    }
+
+    public class DoNothingProcessControlFactory : IProcessControlFactory
+    {
+        public IProcessControl CreateProcessControl(ICodexInstance instance)
+        {
+            return new DoNothingProcessControl();
+        }
+    }
+
+    public class DoNothingProcessControl : IProcessControl
+    {
+        public void DeleteDataDirFolder()
+        {
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            throw new NotImplementedException("Not supported by DoNothingProcessControl");
+        }
+
+        public bool HasCrashed()
+        {
+            return false;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+        }
+    }
+}

ProjectPlugins/CodexClient/StoragePurchaseContract.cs

@@ -1,11 +1,9 @@
-using CodexContractsPlugin;
-using CodexPlugin.Hooks;
-using GethPlugin;
+using CodexClient.Hooks;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IStoragePurchaseContract
     {
@@ -15,8 +13,9 @@ namespace CodexPlugin
         StoragePurchase GetStatus();
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished(ICodexContracts contracts);
+        void WaitForStorageContractFinished();
         void WaitForContractFailed();
+        StoragePurchase GetPurchaseStatus();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -71,7 +70,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished(ICodexContracts contracts)
+        public void WaitForStorageContractFinished()
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -83,13 +82,6 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
-
-            contracts.WaitUntilNextPeriod();
-            contracts.WaitUntilNextPeriod();
-
-            var blocks = 3;
-            Log($"Waiting {blocks} blocks for nodes to process payouts...");
-            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
         }
 
         public void WaitForContractFailed()
@@ -103,9 +95,9 @@ namespace CodexPlugin
             WaitForStorageContractState(timeout, "failed");
         }
 
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        public StoragePurchase GetPurchaseStatus()
         {
-            return codexAccess.GetPurchaseStatus(purchaseId);
+            return codexAccess.GetPurchaseStatus(PurchaseId);
         }
 
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)

ProjectPlugins/CodexClient/TransferSpeeds.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface ITransferSpeeds
     {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,5 +1,5 @@
-using GethPlugin;
-using System.Numerics;
+using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -1,6 +1,6 @@
 using CodexContractsPlugin.Marketplace;
-using GethPlugin;
 using Logging;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -1,5 +1,5 @@
-using GethPlugin;
-using System.Numerics;
+using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -5,8 +5,8 @@ using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexConvertors.Extensions;
-using NethereumWorkflow;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin
 {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,7 +1,7 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
 using BlockchainUtils;
-using GethPlugin;
 using Newtonsoft.Json;
+using Utils;
 
 namespace CodexContractsPlugin.Marketplace
 {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -82,12 +82,12 @@ namespace CodexPlugin
         private void OverwriteOpenApiYaml(string containerApi)
         {
             Log("API compatibility check failed. Updating CodexPlugin...");
-            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
-            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexClient/openapi.yaml. Expected: " + openApiFilePath);
 
             File.Delete(openApiFilePath);
             File.WriteAllText(openApiFilePath, containerApi);
-            Log("CodexPlugin/openapi.yaml has been updated.");
+            Log("CodexClient/openapi.yaml has been updated.");
         }
 
         private string Hash(string file)

ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs

@@ -0,0 +1,156 @@
+using CodexClient;
+using Core;
+using Utils;
+using System.Diagnostics;
+
+namespace CodexPlugin
+{
+    public class BinaryCodexStarter : ICodexStarter
+    {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly static NumberSource numberSource = new NumberSource(1);
+        private readonly static FreePortFinder freePortFinder = new FreePortFinder();
+        private readonly static object _lock = new object();
+        private readonly static string dataParentDir = "codex_disttest_datadirs";
+        private readonly static CodexExePath codexExePath = new CodexExePath();
+
+        static BinaryCodexStarter()
+        {
+            StopAllCodexProcesses();
+            DeleteParentDataDir();
+        }
+
+        public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+        }
+
+        public ICodexInstance[] BringOnline(CodexSetup codexSetup)
+        {
+            lock (_lock)
+            {
+                LogSeparator();
+                Log($"Starting {codexSetup.Describe()}...");
+
+                return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes);
+            }
+        }
+
+        public void Decommission()
+        {
+            lock (_lock)
+            {
+                processControlMap.StopAll();
+            }
+        }
+
+        private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes)
+        {
+            var result = new List<ICodexInstance>();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(StartBinary(startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        private ICodexInstance StartBinary(CodexStartupConfig config)
+        {
+            var name = GetName(config);
+            var dataDir = Path.Combine(dataParentDir, $"datadir_{numberSource.GetNextNumber()}");
+            var pconfig = new CodexProcessConfig(name, freePortFinder, dataDir);
+            Log(pconfig);
+
+            var factory = new CodexProcessRecipe(pconfig, codexExePath);
+            var recipe = factory.Initialize(config);
+
+            var startInfo = new ProcessStartInfo(
+                fileName: recipe.Cmd,
+                arguments: recipe.Args
+            );
+            //startInfo.UseShellExecute = true;
+            startInfo.RedirectStandardOutput = true;
+            startInfo.RedirectStandardError = true;
+
+            var process = Process.Start(startInfo);
+            if (process == null || process.HasExited)
+            {
+                throw new Exception("Failed to start");
+            }
+
+            var local = "localhost";
+            var instance = new CodexInstance(
+                name: name,
+                imageName: "binary",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: new Address("Disc", pconfig.LocalIpAddrs.ToString(), pconfig.DiscPort),
+                apiEndpoint: new Address("Api", "http://" + local, pconfig.ApiPort),
+                listenEndpoint: new Address("Listen", local, pconfig.ListenPort),
+                ethAccount: null,
+                metricsEndpoint: null
+            );
+
+            var pc = new BinaryProcessControl(pluginTools.GetLog(), process, pconfig);
+            processControlMap.Add(instance, pc);
+
+            return instance;
+        }
+
+        private string GetName(CodexStartupConfig config)
+        {
+            if (!string.IsNullOrEmpty(config.NameOverride))
+            {
+                return config.NameOverride + "_" + numberSource.GetNextNumber();
+            }
+            return "codex_" + numberSource.GetNextNumber();
+        }
+
+        private void LogSeparator()
+        {
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void Log(CodexProcessConfig pconfig)
+        {
+            Log(
+                "NodeConfig:Name=" + pconfig.Name +
+                "ApiPort=" + pconfig.ApiPort +
+                "DiscPort=" + pconfig.DiscPort +
+                "ListenPort=" + pconfig.ListenPort +
+                "DataDir=" + pconfig.DataDir
+            );
+        }
+
+        private void Log(string message)
+        {
+            pluginTools.GetLog().Log(message);
+        }
+
+        private static void DeleteParentDataDir()
+        {
+            if (Directory.Exists(dataParentDir))
+            {
+                Directory.Delete(dataParentDir, true);
+            }
+        }
+
+        private static void StopAllCodexProcesses()
+        {
+            var processes = Process.GetProcesses();
+            var codexes = processes.Where(p =>
+                p.ProcessName.ToLowerInvariant() == "codex" &&
+                p.MainModule != null &&
+                p.MainModule.FileName == codexExePath.Get()
+            ).ToArray();
+
+            foreach (var c in codexes)
+            {
+                c.Kill();
+                c.WaitForExit();
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/BinaryProcessControl.cs

@@ -0,0 +1,93 @@
+using System.Diagnostics;
+using CodexClient;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class BinaryProcessControl : IProcessControl
+    {
+        private readonly LogFile logFile;
+        private readonly Process process;
+        private readonly CodexProcessConfig config;
+        private List<string> logBuffer = new List<string>();
+        private readonly object bufferLock = new object();
+        private readonly List<Task> streamTasks = new List<Task>();
+        private bool running;
+
+        public BinaryProcessControl(ILog log, Process process, CodexProcessConfig config)
+        {
+            logFile = log.CreateSubfile(config.Name);
+
+            running = true;
+            this.process = process;
+            this.config = config;
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardOutput)));
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardError)));
+            streamTasks.Add(Task.Run(() => WriteLog()));
+        }
+
+        private void ReadProcessStream(StreamReader reader)
+        {
+            while (running)
+            {
+                var line = reader.ReadLine();
+                if (!string.IsNullOrEmpty(line))
+                {
+                    lock (bufferLock)
+                    {
+                        logBuffer.Add(line);
+                    }
+                }
+            }
+        }
+
+        private void WriteLog()
+        {
+            while (running || logBuffer.Count > 0)
+            {
+                if (logBuffer.Count > 0)
+                {
+                    List<string> lines = null!;
+                    lock (bufferLock)
+                    {
+                        lines = logBuffer;
+                        logBuffer = new List<string>();
+                    }
+
+                    logFile.WriteRawMany(lines);
+                }
+                else Thread.Sleep(100);
+            }
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            if (!Directory.Exists(config.DataDir)) throw new Exception("datadir not found");
+            Directory.Delete(config.DataDir, true);
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            return new DownloadedLog(logFile, config.Name);
+        }
+
+        public bool HasCrashed()
+        {
+            return process.HasExited;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            running = false;
+            process.Kill();
+
+            if (waitTillStopped)
+            {
+                process.WaitForExit();
+                foreach (var t in streamTasks) t.Wait();
+            }
+
+            DeleteDataDirFolder();
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs

@@ -0,0 +1,70 @@
+using CodexClient;
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class CodexContainerProcessControl : IProcessControl
+    {
+        private readonly IPluginTools tools;
+        private readonly RunningPod pod;
+        private readonly Action onStop;
+        private readonly ContainerCrashWatcher crashWatcher;
+
+        public CodexContainerProcessControl(IPluginTools tools, RunningPod pod, Action onStop)
+        {
+            this.tools = tools;
+            this.pod = pod;
+            this.onStop = onStop;
+
+            crashWatcher = tools.CreateWorkflow().CreateCrashWatcher(pod.Containers.Single());
+            crashWatcher.Start();
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            var workflow = tools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            crashWatcher.Stop();
+            onStop();
+            Log("Stopped.");
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            var workflow = tools.CreateWorkflow();
+            return workflow.DownloadContainerLog(pod.Containers.Single());
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            var container = pod.Containers.Single();
+
+            try
+            {
+                var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
+                var dataDir = dataDirVar.Value;
+                var workflow = tools.CreateWorkflow();
+                workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
+                Log("Deleted repo folder.");
+            }
+            catch (Exception e)
+            {
+                Log("Unable to delete repo folder: " + e);
+            }
+        }
+
+        public bool HasCrashed()
+        {
+            return crashWatcher.HasCrashed();
+        }
+
+        private void Log(string message)
+        {
+            tools.GetLog().Log(message);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using CodexClient;
+using CodexContractsPlugin;
 using GethPlugin;
 using KubernetesWorkflow.Types;
 
@@ -9,7 +10,7 @@ namespace CodexPlugin
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
             CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
             RunningPod? discordBotContainer, DeploymentMetadata metadata,
-            String id)
+            string id)
         {
             Id = id;
             CodexInstances = codexInstances;
@@ -20,7 +21,7 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
-        public String Id { get; }
+        public string Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
@@ -29,18 +30,6 @@ namespace CodexPlugin
         public DeploymentMetadata Metadata { get; }
     }
 
-    public class CodexInstance
-    {
-        public CodexInstance(RunningPod pod, DebugInfo info)
-        {
-            Pod = pod;
-            Info = info;
-        }
-
-        public RunningPod Pod { get; }
-        public DebugInfo Info { get; }
-    }
-
     public class DeploymentMetadata
     {
         public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,

ProjectPlugins/CodexPlugin/CodexExePath.cs

@@ -0,0 +1,29 @@
+namespace CodexPlugin
+{
+    public class CodexExePath
+    {
+        private readonly string[] paths = [
+            Path.Combine("d:", "Dev", "nim-codex", "build", "codex.exe"),
+            Path.Combine("c:", "Projects", "nim-codex", "build", "codex.exe")
+        ];
+
+        private string selectedPath = string.Empty;
+
+        public CodexExePath()
+        {
+            foreach (var p in paths)
+            {
+                if (File.Exists(p))
+                {
+                    selectedPath = p;
+                    return;
+                }
+            }
+        }
+
+        public string Get()
+        {
+            return selectedPath;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs

@@ -0,0 +1,45 @@
+using CodexClient;
+using KubernetesWorkflow.Types;
+using Utils;
+
+namespace CodexPlugin
+{
+    public static class CodexInstanceContainerExtension
+    {
+        public static ICodexInstance CreateFromPod(RunningPod pod)
+        {
+            var container = pod.Containers.Single();
+
+            return new CodexInstance(
+                name: container.Name,
+                imageName: container.Recipe.Image,
+                startUtc: container.Recipe.RecipeCreatedUtc,
+                discoveryEndpoint: container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag),
+                apiEndpoint: container.GetAddress(CodexContainerRecipe.ApiPortTag),
+                listenEndpoint: container.GetInternalAddress(CodexContainerRecipe.ListenPortTag),
+                ethAccount: container.Recipe.Additionals.Get<EthAccount>(),
+                metricsEndpoint: GetMetricsEndpoint(container)
+            );
+        }
+
+        // todo: is this needed for the discovery address??
+        //var info = codexAccess.GetPodInfo();
+        //return new Address(
+        //    logName: $"{GetName()}:DiscoveryPort",
+        //    host: info.Ip,
+        //    port: Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag)!.Number
+        //);
+
+        private static Address? GetMetricsEndpoint(RunningContainer container)
+        {
+            try
+            {
+                return container.GetInternalAddress(CodexContainerRecipe.MetricsPortTag);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,53 +0,0 @@
-using CodexPlugin.Hooks;
-using Core;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
-
-namespace CodexPlugin
-{
-    public interface ICodexNodeFactory
-    {
-        CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
-        CrashWatcher CreateCrashWatcher(RunningContainer c);
-    }
-
-    public class CodexNodeFactory : ICodexNodeFactory
-    {
-        private readonly IPluginTools tools;
-        private readonly CodexHooksFactory codexHooksFactory;
-
-        public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
-        {
-            this.tools = tools;
-            this.codexHooksFactory = codexHooksFactory;
-        }
-
-        public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        {
-            var ethAccount = GetEthAccount(access);
-            var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
-
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount, hooks);
-            return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
-        }
-
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount, ICodexNodeHooks hooks)
-        {
-            if (ethAccount == null) return new MarketplaceUnavailable();
-            return new MarketplaceAccess(tools.GetLog(), codexAccess, hooks);
-        }
-
-        private EthAccount? GetEthAccount(CodexAccess access)
-        {
-            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
-            if (ethAccount == null) return null;
-            return ethAccount;
-        }
-
-        public CrashWatcher CreateCrashWatcher(RunningContainer c)
-        {
-            return tools.CreateWorkflow().CreateCrashWatcher(c);
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,25 +1,23 @@
-using Core;
-using KubernetesWorkflow.Types;
-using MetricsPlugin;
+using CodexClient;
+using Core;
 using System.Collections;
+using Utils;
 
 namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
     {
-        void BringOffline(bool waitTillStopped);
+        void Stop(bool waitTillStopped);
         ICodexNode this[int index] { get; }
     }
 
     public class CodexNodeGroup : ICodexNodeGroup
     {
-        private readonly CodexStarter starter;
+        private readonly ICodexNode[] nodes;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(IPluginTools tools, ICodexNode[] nodes)
         {
-            this.starter = starter;
-            Containers = containers;
-            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            this.nodes = nodes;
             Version = new DebugInfoVersion();
         }
 
@@ -31,25 +29,23 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline(bool waitTillStopped)
+        public void Stop(bool waitTillStopped)
         {
-            starter.BringOffline(this, waitTillStopped);
-            // Clear everything. Prevent accidental use.
-            Nodes = Array.Empty<CodexNode>();
-            Containers = null!;
+            foreach (var node in Nodes) node.Stop(waitTillStopped);
         }
 
         public void Stop(CodexNode node, bool waitTillStopped)
         {
-            starter.Stop(node.Pod, waitTillStopped);
-            Nodes = Nodes.Where(n => n != node).ToArray();
-            Containers = Containers.Where(c => c != node.Pod).ToArray();
+            node.Stop(waitTillStopped);
         }
 
-        public RunningPod[] Containers { get; private set; }
-        public CodexNode[] Nodes { get; private set; }
+        public ICodexNode[] Nodes => nodes;
         public DebugInfoVersion Version { get; private set; }
-        public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
+
+        public Address[] GetMetricsScrapeTargets()
+        {
+            return Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray();
+        }
 
         public IEnumerator<ICodexNode> GetEnumerator()
         {
@@ -63,12 +59,11 @@ namespace CodexPlugin
 
         public string Describe()
         {
-            return $"group:[{Containers.Describe()}]";
+            return $"group:[{string.Join(",", Nodes.Select(n => n.GetName()))}]";
         }
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
@@ -79,16 +74,6 @@ namespace CodexPlugin
             }
 
             Version = first;
-            foreach (var node in Nodes) node.Initialize();
-        }
-
-        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
-        {
-            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
-            var access = new CodexAccess(tools, c, watcher);
-            var node = factory.CreateOnlineCodexNode(access, this);
-            node.Awake();
-            return node;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,48 +1,68 @@
-using CodexPlugin.Hooks;
+using CodexClient;
+using CodexClient.Hooks;
 using Core;
-using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public class CodexPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
-        private readonly CodexStarter codexStarter;
+        private const bool UseContainers = true;
+
+        private readonly ICodexStarter codexStarter;
         private readonly IPluginTools tools;
         private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace;
+        private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory();
+        private readonly ProcessControlMap processControlMap = new ProcessControlMap();
+        private readonly CodexWrapper codexWrapper;
 
         public CodexPlugin(IPluginTools tools)
         {
-            codexStarter = new CodexStarter(tools);
             this.tools = tools;
+
+            codexStarter = CreateCodexStarter();
+            codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory);
+        }
+
+        private ICodexStarter CreateCodexStarter()
+        {
+            if (UseContainers)
+            {
+                Log("Using Containerized Codex instances");
+                return new ContainerCodexStarter(tools, processControlMap);
+            }
+
+            Log("Using Binary Codex instances");
+            return new BinaryCodexStarter(tools, processControlMap);
         }
 
         public string LogPrefix => "(Codex) ";
 
         public void Announce()
         {
-            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
-            metadata.Add("codexid", codexStarter.GetCodexId());
-            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
+            metadata.Add("codexid", codexWrapper.GetCodexId());
+            metadata.Add("codexrevision", codexWrapper.GetCodexRevision());
         }
 
         public void Decommission()
         {
+            codexStarter.Decommission();
         }
 
-        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexInstance[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(ICodexInstance[] instances)
         {
-            containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
-            return codexStarter.WrapCodexContainers(coreInterface, containers);
+            instances = instances.Select(c => SerializeGate.Gate(c as CodexInstance)).ToArray();
+            return codexWrapper.WrapCodexInstances(instances);
         }
 
         public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
@@ -62,9 +82,10 @@ namespace CodexPlugin
             }
         }
 
-        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider)
         {
-            codexStarter.HooksFactory.Provider = hooksProvider;
+            if (hooksFactory.Providers.Contains(hooksProvider)) return;
+            hooksFactory.Providers.Add(hooksProvider);
         }
 
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -6,14 +6,6 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <ItemGroup>
-    <None Remove="openapi.yaml" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
       <PrivateAssets>all</PrivateAssets>
@@ -30,6 +22,7 @@
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
+    <ProjectReference Include="..\CodexClient\CodexClient.csproj" />
     <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />

ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs

@@ -0,0 +1,160 @@
+using System.Net.Sockets;
+using System.Net;
+using Nethereum.Util;
+
+namespace CodexPlugin
+{
+    public class ProcessRecipe
+    {
+        public ProcessRecipe(string cmd, string[] args)
+        {
+            Cmd = cmd;
+            Args = args;
+        }
+
+        public string Cmd { get; }
+        public string[] Args { get; }
+    }
+
+    public class CodexProcessConfig
+    {
+        public CodexProcessConfig(string name, FreePortFinder freePortFinder, string dataDir)
+        {
+            ApiPort = freePortFinder.GetNextFreePort();
+            DiscPort = freePortFinder.GetNextFreePort();
+            ListenPort = freePortFinder.GetNextFreePort();
+            Name = name;
+            DataDir = dataDir;
+            var host = Dns.GetHostEntry(Dns.GetHostName());
+            var addrs = host.AddressList.Where(a => a.AddressFamily == AddressFamily.InterNetwork).ToList();
+
+            LocalIpAddrs = addrs.First();
+        }
+
+        public int ApiPort { get; }
+        public int DiscPort { get; }
+        public int ListenPort { get; }
+        public string Name { get; }
+        public string DataDir { get; }
+        public IPAddress LocalIpAddrs { get; }
+    }
+
+    public class CodexProcessRecipe
+    {
+        private readonly CodexProcessConfig pc;
+        private readonly CodexExePath codexExePath;
+
+        public CodexProcessRecipe(CodexProcessConfig pc, CodexExePath codexExePath)
+        {
+            this.pc = pc;
+            this.codexExePath = codexExePath;
+        }
+
+        public ProcessRecipe Initialize(CodexStartupConfig config)
+        {
+            args.Clear();
+            
+            AddArg("--api-port", pc.ApiPort);
+            AddArg("--api-bindaddr", "0.0.0.0");
+
+            AddArg("--data-dir", pc.DataDir);
+
+            AddArg("--disc-port", pc.DiscPort);
+            AddArg("--log-level", config.LogLevelWithTopics());
+
+            // This makes the node announce itself to its local IP address.
+            AddArg("--nat", $"extip:{pc.LocalIpAddrs.ToStringInvariant()}");
+            
+            AddArg("--listen-addrs", $"/ip4/0.0.0.0/tcp/{pc.ListenPort}");
+
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddArg("--bootstrap-node", config.BootstrapSpr);
+            }
+            if (config.StorageQuota != null)
+            {
+                AddArg("--storage-quota", config.StorageQuota.SizeInBytes.ToString()!);
+            }
+            if (config.BlockTTL != null)
+            {
+                AddArg("--block-ttl", config.BlockTTL.ToString()!);
+            }
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddArg("--block-mi", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddArg("--block-mn", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsEnabled)
+            {
+                throw new Exception("Not supported");
+                //var metricsPort = CreateApiPort(config, MetricsPortTag);
+                //AddEnvVar("CODEX_METRICS", "true");
+                //AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
+                //AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                //AddPodAnnotation("prometheus.io/scrape", "true");
+                //AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
+            }
+
+            if (config.SimulateProofFailures != null)
+            {
+                throw new Exception("Not supported");
+                //AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+
+            if (config.MarketplaceConfig != null)
+            {
+                throw new Exception("Not supported");
+                //var mconfig = config.MarketplaceConfig;
+                //var gethStart = mconfig.GethNode.StartResult;
+                //var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
+                //var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
+
+                //AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
+                //AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
+
+                //var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
+
+                //// Custom scripting in the Codex test image will write this variable to a private-key file,
+                //// and pass the correct filename to Codex.
+                //var account = marketplaceSetup.EthAccountSetup.GetNew();
+                //AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
+                //Additional(account);
+
+                //SetCommandOverride(marketplaceSetup);
+                //if (marketplaceSetup.IsValidator)
+                //{
+                //    AddEnvVar("CODEX_VALIDATOR", "true");
+                //}
+            }
+
+            //if (!string.IsNullOrEmpty(config.NameOverride))
+            //{
+            //    AddEnvVar("CODEX_NODENAME", config.NameOverride);
+            //}
+
+            return Create();
+        }
+
+        private ProcessRecipe Create()
+        {
+            return new ProcessRecipe(
+                cmd: codexExePath.Get(),
+                args: args.ToArray());
+        }
+
+        private readonly List<string> args = new List<string>();
+
+        private void AddArg(string arg, string val)
+        {
+            args.Add($"{arg}={val}");
+        }
+
+        private void AddArg(string arg, int val)
+        {
+            args.Add($"{arg}={val}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using CodexClient;
+using CodexContractsPlugin;
 using GethPlugin;
 using KubernetesWorkflow;
 using Utils;
@@ -223,7 +224,7 @@ namespace CodexPlugin
         {
             if (pinned) return accounts.Last();
 
-            var a = EthAccount.GenerateNew();
+            var a = EthAccountGenerator.GenerateNew();
             accounts.Add(a);
             return a;
         }

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using CodexClient;
+using KubernetesWorkflow;
 using Utils;
 
 namespace CodexPlugin

ProjectPlugins/CodexPlugin/CodexWrapper.cs

@@ -0,0 +1,80 @@
+using CodexClient;
+using CodexClient.Hooks;
+using Core;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class CodexWrapper
+    {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly CodexHooksFactory hooksFactory;
+        private DebugInfoVersion? versionResponse;
+
+        public CodexWrapper(IPluginTools pluginTools, ProcessControlMap processControlMap, CodexHooksFactory hooksFactory)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+            this.hooksFactory = hooksFactory;
+        }
+
+        public string GetCodexId()
+        {
+            if (versionResponse != null) return versionResponse.Version;
+            return "unknown";
+        }
+
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.Revision;
+            return "unknown";
+        }
+
+        public ICodexNodeGroup WrapCodexInstances(ICodexInstance[] instances)
+        {
+            var codexNodeFactory = new CodexNodeFactory(
+                log: pluginTools.GetLog(),
+                fileManager: pluginTools.GetFileManager(),
+                hooksFactory: hooksFactory,
+                httpFactory: pluginTools,
+                processControlFactory: processControlMap);
+
+            var group = CreateCodexGroup(instances, codexNodeFactory);
+
+            pluginTools.GetLog().Log($"Codex version: {group.Version}");
+            versionResponse = group.Version;
+
+            return group;
+        }
+
+        private CodexNodeGroup CreateCodexGroup(ICodexInstance[] instances, CodexNodeFactory codexNodeFactory)
+        {
+            var nodes = instances.Select(codexNodeFactory.CreateCodexNode).ToArray();
+            var group = new CodexNodeGroup(pluginTools, nodes);
+
+            try
+            {
+                Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
+            }
+            catch
+            {
+                CodexNodesNotOnline(instances);
+                throw;
+            }
+
+            return group;
+        }
+
+        private void CodexNodesNotOnline(ICodexInstance[] instances)
+        {
+            pluginTools.GetLog().Log("Codex nodes failed to start");
+            var log = pluginTools.GetLog();
+            foreach (var i in instances)
+            {
+                var pc = processControlMap.Get(i);
+                pc.DownloadLog(log.CreateSubfile(i.Name + "_failed_to_start"));
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs

@@ -1,29 +1,26 @@
-using CodexPlugin.Hooks;
+using CodexClient;
 using Core;
-using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
-using Logging;
+using Utils;
 
 namespace CodexPlugin
 {
-    public class CodexStarter
+    public class ContainerCodexStarter : ICodexStarter
     {
         private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
         private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
         private readonly ApiChecker apiChecker;
-        private DebugInfoVersion? versionResponse;
 
-        public CodexStarter(IPluginTools pluginTools)
+        public ContainerCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
         {
             this.pluginTools = pluginTools;
-
+            this.processControlMap = processControlMap;
             apiChecker = new ApiChecker(pluginTools);
         }
 
-        public CodexHooksFactory HooksFactory { get; } = new CodexHooksFactory();
-
-        public RunningPod[] BringOnline(CodexSetup codexSetup)
+        public ICodexInstance[] BringOnline(CodexSetup codexSetup)
         {
             LogSeparator();
             Log($"Starting {codexSetup.Describe()}...");
@@ -43,51 +40,11 @@ namespace CodexPlugin
             }
             LogSeparator();
 
-            return containers;
+            return containers.Select(CreateInstance).ToArray();
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public void Decommission()
         {
-            var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
-
-            var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);
-
-            Log($"Codex version: {group.Version}");
-            versionResponse = group.Version;
-
-            return group;
-        }
-
-        public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
-        {
-            Log($"Stopping {group.Describe()}...");
-            StopCrashWatcher(group);
-            var workflow = pluginTools.CreateWorkflow();
-            foreach (var c in group.Containers)
-            {
-                workflow.Stop(c, waitTillStopped);
-            }
-            Log("Stopped.");
-        }
-
-        public void Stop(RunningPod pod, bool waitTillStopped)
-        {
-            Log($"Stopping node...");
-            var workflow = pluginTools.CreateWorkflow();
-            workflow.Stop(pod, waitTillStopped);
-            Log("Stopped.");
-        }
-
-        public string GetCodexId()
-        {
-            if (versionResponse != null) return versionResponse.Version;
-            return recipe.Image;
-        }
-
-        public string GetCodexRevision()
-        {
-            if (versionResponse != null) return versionResponse.Revision;
-            return "unknown";
         }
 
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
@@ -118,27 +75,15 @@ namespace CodexPlugin
             return workflow.GetPodInfo(rc);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private ICodexInstance CreateInstance(RunningPod pod)
         {
-            var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
-
-            try
+            var instance = CodexInstanceContainerExtension.CreateFromPod(pod);
+            var processControl = new CodexContainerProcessControl(pluginTools, pod, onStop: () =>
             {
-                Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
-            }
-            catch
-            {
-                CodexNodesNotOnline(coreInterface, runningContainers);
-                throw;
-            }
-
-            return group;
-        }
-
-        private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
-        {
-            Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
+                processControlMap.Remove(instance);
+            });
+            processControlMap.Add(instance, processControl);
+            return instance;
         }
 
         private void LogSeparator()
@@ -157,13 +102,5 @@ namespace CodexPlugin
         {
             pluginTools.GetLog().Log(message);
         }
-
-        private void StopCrashWatcher(CodexNodeGroup group)
-        {
-            foreach (var node in group)
-            {
-                node.CrashWatcher.Stop();
-            }
-        }
     }
 }

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -1,19 +1,19 @@
-using CodexPlugin.Hooks;
+using CodexClient;
+using CodexClient.Hooks;
 using Core;
-using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
+        public static ICodexInstance[] DeployCodexNodes(this CoreInterface ci, int number, Action<ICodexSetup> setup)
         {
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningPod[] containers)
+        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, ICodexInstance[] instances)
         {
-            return Plugin(ci).WrapCodexContainers(ci, containers);
+            return Plugin(ci).WrapCodexContainers(instances);
         }
 
         public static ICodexNode StartCodexNode(this CoreInterface ci)
@@ -39,9 +39,9 @@ namespace CodexPlugin
             return ci.StartCodexNodes(number, s => { });
         }
 
-        public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
+        public static void AddCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
         {
-            Plugin(ci).SetCodexHooksProvider(hooksProvider);
+            Plugin(ci).AddCodexHooksProvider(hooksProvider);
         }
 
         private static CodexPlugin Plugin(CoreInterface ci)

ProjectPlugins/CodexPlugin/FreePortFinder.cs

@@ -0,0 +1,43 @@
+using System.Net.NetworkInformation;
+
+namespace CodexPlugin
+{
+    public class FreePortFinder
+    {
+        private readonly object _lock = new object();
+        private int nextPort = 8080;
+
+        public int GetNextFreePort()
+        {
+            lock (_lock)
+            {
+                return Next();
+            }
+        }
+
+        private int Next()
+        {
+            while (true)
+            {
+                var p = nextPort;
+                nextPort++;
+
+                if (!IsInUse(p))
+                {
+                    return p;
+                }
+
+                if (nextPort > 30000) throw new Exception("Running out of ports.");
+            }
+        }
+
+        private bool IsInUse(int port)
+        {
+            var ipProps = IPGlobalProperties.GetIPGlobalProperties();
+            if (ipProps.GetActiveTcpConnections().Any(t => t.LocalEndPoint.Port == port)) return true;
+            if (ipProps.GetActiveTcpListeners().Any(t => t.Port == port)) return true;
+            if (ipProps.GetActiveUdpListeners().Any(u => u.Port == port)) return true;
+            return false;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/Hooks/CodexNodeHooks.cs

@@ -1,19 +0,0 @@
-using GethPlugin;
-using Utils;
-
-namespace CodexPlugin.Hooks
-{
-    public interface ICodexNodeHooks
-    {
-        void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
-        void OnNodeStarted(string peerId, string nodeId);
-        void OnNodeStopping();
-        void OnFileUploading(string uid, ByteSize size);
-        void OnFileUploaded(string uid, ByteSize size, ContentId cid);
-        void OnFileDownloading(ContentId cid);
-        void OnFileDownloaded(ByteSize size, ContentId cid);
-        void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
-        void OnStorageContractUpdated(StoragePurchase purchaseStatus);
-        void OnStorageAvailabilityCreated(StorageAvailability response);
-    }
-}

ProjectPlugins/CodexPlugin/ICodexStarter.cs

@@ -0,0 +1,10 @@
+using CodexClient;
+
+namespace CodexPlugin
+{
+    public interface ICodexStarter
+    {
+        ICodexInstance[] BringOnline(CodexSetup codexSetup);
+        void Decommission();
+    }
+}

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs

@@ -1,5 +1,6 @@
-using CodexPlugin.OverwatchSupport.LineConverters;
-using KubernetesWorkflow;
+using CodexClient;
+using CodexPlugin.OverwatchSupport.LineConverters;
+using Logging;
 using OverwatchTranscript;
 using Utils;
 

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs

@@ -1,5 +1,5 @@
-using CodexPlugin.Hooks;
-using GethPlugin;
+using CodexClient;
+using CodexClient.Hooks;
 using OverwatchTranscript;
 using Utils;
 
@@ -32,7 +32,7 @@ namespace CodexPlugin.OverwatchSupport
             });
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
             if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
             {

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs

@@ -1,5 +1,4 @@
-using CodexPlugin.Hooks;
-using KubernetesWorkflow;
+using CodexClient.Hooks;
 using Logging;
 using OverwatchTranscript;
 using Utils;

ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs

@@ -1,4 +1,6 @@
-namespace CodexPlugin.OverwatchSupport
+using CodexClient;
+
+namespace CodexPlugin.OverwatchSupport
 {
     public class IdentityMap
     {

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs

@@ -1,4 +1,6 @@
-namespace CodexPlugin.OverwatchSupport.LineConverters
+using CodexClient;
+
+namespace CodexPlugin.OverwatchSupport.LineConverters
 {
     public class BlockReceivedLineConverter : ILineConverter
     {

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs

@@ -1,4 +1,6 @@
-namespace CodexPlugin.OverwatchSupport.LineConverters
+using CodexClient;
+
+namespace CodexPlugin.OverwatchSupport.LineConverters
 {
     public class BootstrapLineConverter : ILineConverter
     {

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs

@@ -1,4 +1,6 @@
-namespace CodexPlugin.OverwatchSupport.LineConverters
+using CodexClient;
+
+namespace CodexPlugin.OverwatchSupport.LineConverters
 {
     public class DialSuccessfulLineConverter : ILineConverter
     {

ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs

@@ -1,4 +1,6 @@
-namespace CodexPlugin.OverwatchSupport.LineConverters
+using CodexClient;
+
+namespace CodexPlugin.OverwatchSupport.LineConverters
 {
     public class PeerDroppedLineConverter : ILineConverter
     {

ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs

@@ -1,4 +1,5 @@
-using OverwatchTranscript;
+using CodexClient;
+using OverwatchTranscript;
 
 namespace CodexPlugin.OverwatchSupport
 {

ProjectPlugins/CodexPlugin/ProcessControlMap.cs

@@ -0,0 +1,37 @@
+using CodexClient;
+
+namespace CodexPlugin
+{
+    public class ProcessControlMap : IProcessControlFactory
+    {
+        private readonly Dictionary<string, IProcessControl> processControlMap = new Dictionary<string, IProcessControl>();
+
+        public void Add(ICodexInstance instance, IProcessControl control)
+        {
+            processControlMap.Add(instance.Name, control);
+        }
+
+        public void Remove(ICodexInstance instance)
+        {
+            processControlMap.Remove(instance.Name);
+        }
+
+        public IProcessControl CreateProcessControl(ICodexInstance instance)
+        {
+            return Get(instance);
+        }
+
+        public IProcessControl Get(ICodexInstance instance)
+        {
+            return processControlMap[instance.Name];
+        }
+
+        public void StopAll()
+        {
+            var pcs = processControlMap.Values.ToArray();
+            processControlMap.Clear();
+
+            foreach (var c in pcs) c.Stop(waitTillStopped: true);
+        }
+    }
+}

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -12,11 +12,12 @@ public static class Program
     {
         Console.WriteLine("Injecting hash of 'openapi.yaml'...");
 
-        var root = FindCodexPluginFolder();
-        Console.WriteLine("Located CodexPlugin: " + root);
-        var openApiFile = Path.Combine(root, "openapi.yaml");
-        var clientFile = Path.Combine(root, "obj", "openapiClient.cs");
-        var targetFile = Path.Combine(root, "ApiChecker.cs");
+        var pluginRoot = FindCodexPluginFolder();
+        var clientRoot = FindCodexClientFolder();
+        Console.WriteLine("Located CodexPlugin: " + pluginRoot);
+        var openApiFile = Path.Combine(clientRoot, "openapi.yaml");
+        var clientFile = Path.Combine(clientRoot, "obj", "openapiClient.cs");
+        var targetFile = Path.Combine(pluginRoot, "ApiChecker.cs");
 
         // Force client rebuild by deleting previous artifact.
         File.Delete(clientFile);
@@ -46,6 +47,13 @@ public static class Program
         return folder;
     }
 
+    private static string FindCodexClientFolder()
+    {
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient");
+        if (!Directory.Exists(folder)) throw new Exception("CodexClient folder not found. Expected: " + folder);
+        return folder;
+    }
+
     private static string CreateHash(string openApiFile)
     {
         var file = File.ReadAllText(openApiFile);

ProjectPlugins/GethPlugin/EthAccountGenerator.cs

@@ -1,20 +1,11 @@
 using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Web3.Accounts;
+using Utils;
 
 namespace GethPlugin
 {
-    [Serializable]
-    public class EthAccount
+    public static class EthAccountGenerator
     {
-        public EthAccount(EthAddress ethAddress, string privateKey)
-        {
-            EthAddress = ethAddress;
-            PrivateKey = privateKey;
-        }
-
-        public EthAddress EthAddress { get; }
-        public string PrivateKey { get; }
-
         public static EthAccount GenerateNew()
         {
             var ecKey = Nethereum.Signer.EthECKey.GenerateKey();
@@ -24,10 +15,5 @@ namespace GethPlugin
 
             return new EthAccount(ethAddress, account.PrivateKey);
         }
-
-        public override string ToString()
-        {
-            return EthAddress.ToString();
-        }
     }
 }

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -1,6 +1,7 @@
 using Core;
 using KubernetesWorkflow.Types;
 using Logging;
+using Utils;
 
 namespace MetricsPlugin
 {
@@ -8,35 +9,35 @@ namespace MetricsPlugin
     {
         public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.GetMetricsScrapeTarget()).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params Address[] scrapeTargets)
         {
             return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
         {
-            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget);
+            return ci.WrapMetricsCollector(metricsPod, scrapeTarget.GetMetricsScrapeTarget());
         }
 
-        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IMetricsScrapeTarget scrapeTarget)
+        public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, Address scrapeTarget)
         {
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.GetMetricsScrapeTargets()).ToArray());
         }
 
         public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.GetMetricsScrapeTarget()).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params Address[] scrapeTargets)
         {
             var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -15,13 +15,13 @@ namespace MetricsPlugin
     public class MetricsAccess : IMetricsAccess
     {
         private readonly MetricsQuery query;
-        private readonly IMetricsScrapeTarget target;
+        private readonly Address target;
 
-        public MetricsAccess(MetricsQuery query, IMetricsScrapeTarget target)
+        public MetricsAccess(MetricsQuery query, Address target)
         {
             this.query = query;
             this.target = target;
-            TargetName = target.Container.Name;
+            TargetName = $"'{target.Host}'";
         }
 
         public string TargetName { get; }

ProjectPlugins/MetricsPlugin/MetricsDownloader.cs

@@ -26,7 +26,7 @@ namespace MetricsPlugin
         private LogFile WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
         {
             var file = log.CreateSubfile("csv");
-            log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
+            log.Log($"Downloading metrics for {nodeName} to file {file.Filename}");
 
             file.WriteRaw(string.Join(",", headers));
 

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -1,6 +1,7 @@
 using Core;
 using KubernetesWorkflow.Types;
 using Logging;
+using Utils;
 
 namespace MetricsPlugin
 {
@@ -31,12 +32,12 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
+        public RunningPod DeployMetricsCollector(Address[] scrapeTargets, TimeSpan scrapeInterval)
         {
             return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
-        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)
+        public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, Address target)
         {
             runningPod = SerializeGate.Gate(runningPod);
             return starter.CreateAccessForTarget(runningPod, target);

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -3,6 +3,8 @@ using IdentityModel;
 using KubernetesWorkflow.Types;
 using Logging;
 using System.Globalization;
+using Utils;
+using WebUtils;
 
 namespace MetricsPlugin
 {
@@ -23,7 +25,7 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, Address target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
             if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
@@ -53,7 +55,7 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(Address target)
         {
             var instanceString = GetInstanceStringForNode(target);
             var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
@@ -139,12 +141,12 @@ namespace MetricsPlugin
             };
         }
 
-        private string GetInstanceNameForNode(IMetricsScrapeTarget target)
+        private string GetInstanceNameForNode(Address target)
         {
-            return ScrapeTargetHelper.FormatTarget(log, target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
 
-        private string GetInstanceStringForNode(IMetricsScrapeTarget target)
+        private string GetInstanceStringForNode(Address target)
         {
             return "{instance=\"" + GetInstanceNameForNode(target) + "\"}";
         }
@@ -172,9 +174,9 @@ namespace MetricsPlugin
             return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
         }
 
-        private void Log(IMetricsScrapeTarget target, string metricName, Metrics result)
+        private void Log(Address target, string metricName, Metrics result)
         {
-            Log($"{target.Container.Name} '{metricName}' = {result}");
+            Log($"{target.LogName} '{metricName}' = {result}");
         }
 
         private void Log(string metricName, Metrics result)
@@ -182,9 +184,9 @@ namespace MetricsPlugin
             Log($"'{metricName}' = {result}");
         }
 
-        private void Log(IMetricsScrapeTarget target, Metrics result)
+        private void Log(Address target, Metrics result)
         {
-            Log($"{target.Container.Name} => {result}");
+            Log($"{target.LogName} => {result}");
         }
 
         private void Log(string msg)

ProjectPlugins/MetricsPlugin/MetricsScrapeTarget.cs

@@ -1,32 +0,0 @@
-using KubernetesWorkflow.Types;
-
-namespace MetricsPlugin
-{
-    public interface IMetricsScrapeTarget
-    {
-        RunningContainer Container { get; }
-        string MetricsPortTag { get; }
-    }
-
-    public interface IHasMetricsScrapeTarget
-    {
-        IMetricsScrapeTarget MetricsScrapeTarget { get; }
-    }
-
-    public interface IHasManyMetricScrapeTargets
-    {
-        IMetricsScrapeTarget[] ScrapeTargets { get; }
-    }
-
-    public class MetricsScrapeTarget : IMetricsScrapeTarget
-    {
-        public MetricsScrapeTarget(RunningContainer container, string metricsPortTag)
-        {
-            Container = container;
-            MetricsPortTag = metricsPortTag;
-        }
-
-        public RunningContainer Container { get; }
-        public string MetricsPortTag { get; }
-    }
-}

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -3,6 +3,7 @@ using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
 using System.Text;
+using Utils;
 
 namespace MetricsPlugin
 {
@@ -16,7 +17,7 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
+        public RunningPod CollectMetricsFor(Address[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
@@ -32,7 +33,7 @@ namespace MetricsPlugin
             return runningContainers;
         }
 
-        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, IMetricsScrapeTarget target)
+        public MetricsAccess CreateAccessForTarget(RunningPod metricsPod, Address target)
         {
             var metricsQuery = new MetricsQuery(tools, metricsPod.Containers.Single());
             return new MetricsAccess(metricsQuery, target);
@@ -48,7 +49,7 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
+        private string GeneratePrometheusConfig(Address[] targets, TimeSpan scrapeInterval)
         {
             var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
             if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
@@ -74,19 +75,18 @@ namespace MetricsPlugin
             return Convert.ToBase64String(bytes);
         }
 
-        private string FormatTarget(IMetricsScrapeTarget target)
+        private string FormatTarget(Address target)
         {
-            return ScrapeTargetHelper.FormatTarget(tools.GetLog(), target);
+            return ScrapeTargetHelper.FormatTarget(target);
         }
     }
 
     public static class ScrapeTargetHelper
     {
-        public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
+        public static string FormatTarget(Address target)
         {
-            var a = target.Container.GetAddress(target.MetricsPortTag);
-            var host = a.Host.Replace("http://", "").Replace("https://", "");
-            return $"{host}:{a.Port}";
+            var host = target.Host.Replace("http://", "").Replace("https://", "");
+            return $"{host}:{target.Port}";
         }
     }
 }

Tests/CodexContinuousTests/ContinuousTest.cs

@@ -1,15 +1,13 @@
-using CodexPlugin;
-using Core;
-using DistTestCore;
+using CodexClient;
 using FileUtils;
 using Logging;
 using MetricsPlugin;
+using Utils;
 
 namespace ContinuousTests
 {
     public abstract class ContinuousTestLongTimeouts : ContinuousTest
     {
-        public override ITimeSet TimeSet => new LongTimeSet();
     }
 
     public abstract class ContinuousTest
@@ -44,16 +42,15 @@ namespace ContinuousTests
         public ILog Log { get; private set; } = null!;
         public IFileManager FileManager { get; private set; } = null!;
         public Configuration Configuration { get; private set; } = null!;
-        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
         public CancellationToken CancelToken { get; private set; } = new CancellationToken();
         public NodeRunner NodeRunner { get; private set; } = null!;
 
         public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target)
         {
-            return CreateMetricsAccess(target.MetricsScrapeTarget);
+            return CreateMetricsAccess(target.GetMetricsScrapeTarget());
         }
 
-        public IMetricsAccess CreateMetricsAccess(IMetricsScrapeTarget target)
+        public IMetricsAccess CreateMetricsAccess(Address target)
         {
             if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment.");
 

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -2,6 +2,7 @@
 using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
+using WebUtils;
 
 namespace ContinuousTests
 {
@@ -37,7 +38,7 @@ namespace ContinuousTests
             var endpoint = CreateElasticSearchEndpoint();
             var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc);
 
-            targetFile.Write($"Downloading '{container.Name}' to '{targetFile.FullFilename}'.");
+            targetFile.Write($"Downloading '{container.Name}' to '{targetFile.Filename}'.");
             var reconstructor = new LogReconstructor(targetFile, endpoint, queryTemplate);
             reconstructor.DownloadFullLog();
 
@@ -68,7 +69,7 @@ namespace ContinuousTests
         {
             var serviceName = "elasticsearch";
             var k8sNamespace = "monitoring";
-            var address = new Address($"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
+            var address = new Address("ElasticSearchEndpoint", $"http://{serviceName}.{k8sNamespace}.svc.cluster.local", 9200);
             var baseUrl = "";
 
             var http = tools.CreateHttp(address.ToString(), client =>

Tests/CodexContinuousTests/NodeRunner.cs

@@ -3,8 +3,7 @@ using Logging;
 using Utils;
 using Core;
 using CodexPlugin;
-using KubernetesWorkflow.Types;
-using KubernetesWorkflow;
+using CodexClient;
 
 namespace ContinuousTests
 {
@@ -24,12 +23,6 @@ namespace ContinuousTests
             this.customNamespace = customNamespace;
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
-        {
-            var entryPoint = CreateEntryPoint();
-            return entryPoint.CreateInterface().DownloadLog(container, tailLines);
-        }
-
         public void RunNode(Action<ICodexSetup> setup, Action<ICodexNode> operation)
         {
             RunNode(nodes.ToList().PickOneRandom(), setup, operation);
@@ -40,7 +33,7 @@ namespace ContinuousTests
             var entryPoint = CreateEntryPoint();
             // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network.
             // Therefore, we use the image of the bootstrap node.
-            CodexContainerRecipe.DockerImageOverride = bootstrapNode.Container.Recipe.Image;
+            CodexContainerRecipe.DockerImageOverride = bootstrapNode.GetImageName();
 
             try
             {
@@ -59,7 +52,7 @@ namespace ContinuousTests
                 }
                 catch
                 {
-                    DownloadLog(node.Container);
+                    node.DownloadLog();
                     throw;
                 }
             }

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -7,6 +7,7 @@ using DistTestCore.Logs;
 using Core;
 using KubernetesWorkflow.Types;
 using TaskFactory = Utils.TaskFactory;
+using CodexClient;
 
 namespace ContinuousTests
 {
@@ -125,13 +126,14 @@ namespace ContinuousTests
 
             foreach (var node in nodes)
             {
-                var container = node.Container;
-                var deploymentName = container.RunningPod.StartResult.Deployment.Name;
-                var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
-                var openingLine =
-                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
-                    effectiveEnd, openingLine);
+                //var container = node.Container;
+                //var deploymentName = container.RunningPod.StartResult.Deployment.Name;
+                //var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
+                //var openingLine =
+                //    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
+                //elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
+                //    effectiveEnd, openingLine);
+                throw new NotImplementedException("access to container info is unavilable.");
             }
         }
 
@@ -204,7 +206,6 @@ namespace ContinuousTests
             result.Add("testname", testName);
             result.Add("message", message);
             result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
-            result.Add("involvedpodnames", string.Join(",", nodes.Select(n => n.GetPodInfo().Name)));
 
             var error = message.Split(Environment.NewLine).First();
             if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
@@ -286,26 +287,26 @@ namespace ContinuousTests
         private string GetContainerNames()
         {
             if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
-            return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+            return $"({string.Join(",", nodes.Select(n => n.GetName()))})";
         }
 
         private ICodexNode[] CreateRandomNodes()
         {
-            var containers = SelectRandomContainers();
-            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
-            return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
+            var instances = SelectRandomInstance();
+            fixtureLog.Log("Selected nodes: " + string.Join(",", instances.Select(c => c.Name)));
+            return entryPoint.CreateInterface().WrapCodexContainers(instances).ToArray();
         }
 
-        private RunningPod[] SelectRandomContainers()
+        private ICodexInstance[] SelectRandomInstance()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToList();
-            if (number == -1) return containers.ToArray();
+            var instances = config.CodexDeployment.CodexInstances.ToList();
+            if (number == -1) return instances.ToArray();
 
-            var result = new RunningPod[number];
+            var result = new ICodexInstance[number];
             for (var i = 0; i < number; i++)
             {
-                result[i] = containers.PickOneRandom();
+                result[i] = instances.PickOneRandom();
             }
 
             return result;

Tests/CodexContinuousTests/StartupChecker.cs

@@ -1,8 +1,7 @@
-using CodexPlugin;
+using CodexClient;
 using Core;
 using DistTestCore.Logs;
 using Logging;
-using Newtonsoft.Json;
 
 namespace ContinuousTests
 {
@@ -39,25 +38,27 @@ namespace ContinuousTests
         private void IncludeDeploymentConfiguration(ILog log)
         {
             log.Log("");
-            var deployment = config.CodexDeployment;
-            var workflow = entryPoint.Tools.CreateWorkflow();
-            foreach (var instance in deployment.CodexInstances)
-            {
-                foreach (var container in instance.Pod.Containers)
-                {
-                    var podInfo = workflow.GetPodInfo(container);
-                    log.Log($"Codex environment variables for '{container.Name}':");
-                    log.Log(
-                        $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
-                    var codexVars = container.Recipe.EnvVars;
-                    foreach (var vars in codexVars) log.Log(vars.ToString());
-                    log.Log("");
-                }
-            }
 
-            log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
-            log.Log("");
+            throw new NotImplementedException();
+            //var deployment = config.CodexDeployment;
+            //var workflow = entryPoint.Tools.CreateWorkflow();
+            //foreach (var instance in deployment.CodexInstances)
+            //{
+            //    foreach (var container in instance.Pod.Containers)
+            //    {
+            //        var podInfo = workflow.GetPodInfo(container);
+            //        log.Log($"Codex environment variables for '{container.Name}':");
+            //        log.Log(
+            //            $"Namespace: {container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+            //            $"Pod name: {podInfo.Name} - Deployment name: {instance.Pod.StartResult.Deployment.Name}");
+            //        var codexVars = container.Recipe.EnvVars;
+            //        foreach (var vars in codexVars) log.Log(vars.ToString());
+            //        log.Log("");
+            //    }
+            //}
+
+            //log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
+            //log.Log("");
         }
 
         private void PreflightCheck(Configuration config)
@@ -91,31 +92,33 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
-            var pass = true;
-            foreach (var n in nodes)
-            {
-                cancelToken.ThrowIfCancellationRequested();
+            throw new NotImplementedException();
 
-                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
-                log.Log($"Checking {n.Container.Name} @ '{address}'...");
+            //var nodes = entryPoint.CreateInterface()
+            //    .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray());
+            //var pass = true;
+            //foreach (var n in nodes)
+            //{
+            //    cancelToken.ThrowIfCancellationRequested();
 
-                if (EnsureOnline(log, n))
-                {
-                    log.Log("OK");
-                }
-                else
-                {
-                    log.Error($"No response from '{address}'.");
-                    pass = false;
-                }
-            }
+            //    var address = n.GetApiEndpoint();
+            //    log.Log($"Checking {n.GetName()} @ '{address}'...");
 
-            if (!pass)
-            {
-                throw new Exception("Not all codex nodes responded.");
-            }
+            //    if (EnsureOnline(log, n))
+            //    {
+            //        log.Log("OK");
+            //    }
+            //    else
+            //    {
+            //        log.Error($"No response from '{address}'.");
+            //        pass = false;
+            //    }
+            //}
+
+            //if (!pass)
+            //{
+            //    throw new Exception("Not all codex nodes responded.");
+            //}
         }
 
         private bool EnsureOnline(BaseLog log, ICodexNode n)

Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using FileUtils;
 using NUnit.Framework;
 using Utils;

Tests/CodexContinuousTests/Tests/PeersTest.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using CodexTests.Helpers;
 using ContinuousTests;
 using NUnit.Framework;

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using FileUtils;
 using Logging;
 using NUnit.Framework;

Tests/CodexLongTests/BasicTests/DownloadTests.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using CodexTests;
 using DistTestCore;
 using FileUtils;

Tests/CodexLongTests/BasicTests/UploadTests.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using CodexTests;
 using DistTestCore;
 using FileUtils;

Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using DistTestCore;
 using NUnit.Framework;
 using Utils;
@@ -31,7 +31,7 @@ namespace CodexTests.ScalabilityTests
             Task.WaitAll(uploadTasks);
             var cid = new ContentId(uploadTasks.Select(t => t.Result.Id).Distinct().Single());
 
-            var uploadLog = Ci.DownloadLog(hosts[0]);
+            var uploadLog = hosts[0].DownloadLog();
             var expectedNumberOfBlocks = RoundUp(fileSize.MB().SizeInBytes, 64.KB().SizeInBytes) + 1; // +1 for manifest block.
             var blockCids = uploadLog
                 .FindLinesThatContain("Block Stored")
@@ -50,7 +50,7 @@ namespace CodexTests.ScalabilityTests
             var resultFile = client.DownloadContent(cid);
             resultFile!.AssertIsEqual(file);
 
-            var downloadLog = Ci.DownloadLog(client);
+            var downloadLog = client.DownloadLog();
             var host = string.Empty;
             var blockAddressHostMap = new Dictionary<string, List<string>>();
             downloadLog

Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using DistTestCore;
 using FileUtils;
 using NUnit.Framework;
@@ -38,16 +38,16 @@ public class ScalabilityTests : CodexDistTest
         var testFile = GenerateTestFile(fileSizeInMb.MB());
 
         LogNodeStatus(uploader);
-        var contentId = uploader.UploadFile(testFile, f => LogNodeStatus(uploader));
+        var contentId = uploader.UploadFile(testFile);
         LogNodeStatus(uploader);
 
         LogNodeStatus(downloader);
-        var downloadedFile = downloader.DownloadContent(contentId, f => LogNodeStatus(downloader));
+        var downloadedFile = downloader.DownloadContent(contentId);
         LogNodeStatus(downloader);
 
         downloadedFile!.AssertIsEqual(testFile);
 
-        uploader.DeleteRepoFolder();
+        uploader.DeleteDataDirFolder();
         uploader.Stop(true);
 
         var otherDownloader = nodes.PickOneRandom();
@@ -55,8 +55,8 @@ public class ScalabilityTests : CodexDistTest
 
         downloadedFile!.AssertIsEqual(testFile);
 
-        downloader.DeleteRepoFolder();
-        otherDownloader.DeleteRepoFolder();
+        downloader.DeleteDataDirFolder();
+        otherDownloader.DeleteDataDirFolder();
     }
 
     /// <summary>