working two-client test with binary instances

ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs

@@ -1,12 +1,89 @@
 using CodexClient;
+using Core;
+using Utils;
+using System.Diagnostics;
 
 namespace CodexPlugin
 {
     public class BinaryCodexStarter : ICodexStarter
     {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly NumberSource numberSource = new NumberSource(1);
+        private readonly FreePortFinder freePortFinder = new FreePortFinder();
+
+        public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+        }
+
         public ICodexInstance[] BringOnline(CodexSetup codexSetup)
         {
-            throw new NotImplementedException();
+            LogSeparator();
+            Log($"Starting {codexSetup.Describe()}...");
+
+            return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes);
+        }
+
+        private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes)
+        {
+            var result = new List<ICodexInstance>();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(StartBinary(startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        private ICodexInstance StartBinary(CodexStartupConfig config)
+        {
+            var portMap = new CodexPortMap(freePortFinder);
+            var factory = new CodexProcessRecipe(portMap, numberSource);
+            var recipe = factory.Initialize(config);
+
+            var name = "codex_" + numberSource.GetNextNumber();
+            var startInfo = new ProcessStartInfo(
+                fileName: recipe.Cmd,
+                arguments: recipe.Args
+            );
+            //startInfo.UseShellExecute = true;
+            startInfo.RedirectStandardOutput = true;
+            startInfo.RedirectStandardError = true;
+
+            var process = Process.Start(startInfo);
+            if (process == null || process.HasExited)
+            {
+                throw new Exception("Failed to start");
+            }
+
+            var local = "localhost";
+            var instance = new CodexInstance(
+                name: name,
+                imageName: "binary",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: new Address("Disc", local, portMap.DiscPort),
+                apiEndpoint: new Address("Api", "http://" + local, portMap.ApiPort),
+                listenEndpoint: new Address("Listen", local, portMap.ListenPort),
+                ethAccount: null,
+                metricsEndpoint: null
+            );
+
+            var pc = new BinaryProcessControl(process, name);
+            processControlMap.Add(instance, pc);
+
+            return instance;
+        }
+
+        private void LogSeparator()
+        {
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void Log(string message)
+        {
+            pluginTools.GetLog().Log(message);
         }
     }
 }

ProjectPlugins/CodexPlugin/BinaryProcessControl.cs

@@ -0,0 +1,56 @@
+using System.Diagnostics;
+using CodexClient;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class BinaryProcessControl : IProcessControl
+    {
+        private Process process;
+        private readonly string nodeName;
+        private bool running;
+        private readonly List<string> logLines = new List<string>();
+        private readonly List<Task> streamTasks = new List<Task>();
+
+        public BinaryProcessControl(Process process, string nodeName)
+        {
+            running = true;
+            this.process = process;
+            this.nodeName = nodeName;
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardOutput)));
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardError)));
+        }
+
+        private void ReadProcessStream(StreamReader reader)
+        {
+            while (running)
+            {
+                var line = reader.ReadLine();
+                if (!string.IsNullOrEmpty(line)) logLines.Add(line);
+            }
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            throw new NotImplementedException();
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            foreach (var line in logLines) file.WriteRaw(line);
+            return new DownloadedLog(file, nodeName);
+        }
+
+        public bool HasCrashed()
+        {
+            return false;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            running = false;
+            process.Kill();
+            foreach (var t in streamTasks) t.Wait();
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -15,7 +15,8 @@ namespace CodexPlugin
 
         public CodexPlugin(IPluginTools tools)
         {
-            codexStarter = new ContainerCodexStarter(tools, processControlMap);
+            //codexStarter = new ContainerCodexStarter(tools, processControlMap);
+            codexStarter = new BinaryCodexStarter(tools, processControlMap);
             codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory);
             this.tools = tools;
         }

ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs

@@ -0,0 +1,163 @@
+using System.Net.Sockets;
+using System.Net;
+using Utils;
+using Nethereum.Util;
+
+namespace CodexPlugin
+{
+    public class ProcessRecipe
+    {
+        public ProcessRecipe(string cmd, string[] args)
+        {
+            Cmd = cmd;
+            Args = args;
+        }
+
+        public string Cmd { get; }
+        public string[] Args { get; }
+    }
+
+    public class CodexPortMap
+    {
+        public CodexPortMap(FreePortFinder freePortFinder)
+        {
+            ApiPort = freePortFinder.GetNextFreePort();
+            DiscPort = freePortFinder.GetNextFreePort();
+            ListenPort = freePortFinder.GetNextFreePort();
+        }
+
+        public int ApiPort { get; }
+        public int DiscPort { get; }
+        public int ListenPort { get; }
+    }
+
+    public class CodexProcessRecipe
+    {
+        private readonly CodexPortMap portMap;
+        private readonly NumberSource numberSource;
+
+        public CodexProcessRecipe(CodexPortMap portMap, NumberSource numberSource)
+        {
+            this.portMap = portMap;
+            this.numberSource = numberSource;
+        }
+
+        public ProcessRecipe Initialize(CodexStartupConfig config)
+        {
+            args.Clear();
+            
+            AddArg("--api-port", portMap.ApiPort);
+            AddArg("--api-bindaddr", "0.0.0.0");
+
+            var dataDir = $"datadir_{numberSource.GetNextNumber()}";
+            AddArg("--data-dir", dataDir);
+
+            AddArg("--disc-port", portMap.DiscPort);
+            AddArg("--log-level", config.LogLevelWithTopics());
+
+            // This makes the node announce itself to its local IP address.
+            var host = Dns.GetHostEntry(Dns.GetHostName());
+            var addrs = host.AddressList.Where(a => a.AddressFamily == AddressFamily.InterNetwork).ToList();
+            var ipaddrs = addrs.First();
+
+            AddArg("--nat", $"extip:{ipaddrs.ToStringInvariant()}");
+            
+            AddArg("--listen-addrs", $"/ip4/0.0.0.0/tcp/{portMap.ListenPort}");
+
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddArg("--bootstrap-node", config.BootstrapSpr);
+            }
+            if (config.StorageQuota != null)
+            {
+                AddArg("--storage-quota", config.StorageQuota.SizeInBytes.ToString()!);
+            }
+            if (config.BlockTTL != null)
+            {
+                AddArg("--block-ttl", config.BlockTTL.ToString()!);
+            }
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddArg("--block-mi", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddArg("--block-mn", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsEnabled)
+            {
+                throw new Exception("Not supported");
+                //var metricsPort = CreateApiPort(config, MetricsPortTag);
+                //AddEnvVar("CODEX_METRICS", "true");
+                //AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
+                //AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                //AddPodAnnotation("prometheus.io/scrape", "true");
+                //AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
+            }
+
+            if (config.SimulateProofFailures != null)
+            {
+                throw new Exception("Not supported");
+                //AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+
+            if (config.MarketplaceConfig != null)
+            {
+                throw new Exception("Not supported");
+                //var mconfig = config.MarketplaceConfig;
+                //var gethStart = mconfig.GethNode.StartResult;
+                //var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
+                //var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
+
+                //AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
+                //AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
+
+                //var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
+
+                //// Custom scripting in the Codex test image will write this variable to a private-key file,
+                //// and pass the correct filename to Codex.
+                //var account = marketplaceSetup.EthAccountSetup.GetNew();
+                //AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
+                //Additional(account);
+
+                //SetCommandOverride(marketplaceSetup);
+                //if (marketplaceSetup.IsValidator)
+                //{
+                //    AddEnvVar("CODEX_VALIDATOR", "true");
+                //}
+            }
+
+            //if (!string.IsNullOrEmpty(config.NameOverride))
+            //{
+            //    AddEnvVar("CODEX_NODENAME", config.NameOverride);
+            //}
+
+            return Create();
+        }
+
+        private ProcessRecipe Create()
+        {
+            return new ProcessRecipe(
+                cmd: Path.Combine(
+                    "d:",
+                    "Dev",
+                    "nim-codex",
+                    "build",
+                    "codex.exe"
+                ),
+                args: args.ToArray());
+        }
+
+        private readonly List<string> args = new List<string>();
+
+        private void AddArg(string arg, string val)
+        {
+            args.Add($"{arg}={val}");
+        }
+
+        private void AddArg(string arg, int val)
+        {
+            args.Add($"{arg}={val}");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/FreePortFinder.cs

@@ -0,0 +1,43 @@
+using System.Net.NetworkInformation;
+
+namespace CodexPlugin
+{
+    public class FreePortFinder
+    {
+        private readonly object _lock = new object();
+        private int nextPort = 8080;
+
+        public int GetNextFreePort()
+        {
+            lock (_lock)
+            {
+                return Next();
+            }
+        }
+
+        private int Next()
+        {
+            while (true)
+            {
+                var p = nextPort;
+                nextPort++;
+
+                if (!IsInUse(p))
+                {
+                    return p;
+                }
+
+                if (nextPort > 30000) throw new Exception("Running out of ports.");
+            }
+        }
+
+        private bool IsInUse(int port)
+        {
+            var ipProps = IPGlobalProperties.GetIPGlobalProperties();
+            if (ipProps.GetActiveTcpConnections().Any(t => t.LocalEndPoint.Port == port)) return true;
+            if (ipProps.GetActiveTcpListeners().Any(t => t.Port == port)) return true;
+            if (ipProps.GetActiveUdpListeners().Any(u => u.Port == port)) return true;
+            return false;
+        }
+    }
+}