diff --git a/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs b/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs index 4353c818..365c2e8b 100644 --- a/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs @@ -1,12 +1,89 @@ using CodexClient; +using Core; +using Utils; +using System.Diagnostics; namespace CodexPlugin { public class BinaryCodexStarter : ICodexStarter { + private readonly IPluginTools pluginTools; + private readonly ProcessControlMap processControlMap; + private readonly NumberSource numberSource = new NumberSource(1); + private readonly FreePortFinder freePortFinder = new FreePortFinder(); + + public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap) + { + this.pluginTools = pluginTools; + this.processControlMap = processControlMap; + } + public ICodexInstance[] BringOnline(CodexSetup codexSetup) { - throw new NotImplementedException(); + LogSeparator(); + Log($"Starting {codexSetup.Describe()}..."); + + return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes); + } + + private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes) + { + var result = new List(); + for (var i = 0; i < numberOfNodes; i++) + { + result.Add(StartBinary(startupConfig)); + } + + return result.ToArray(); + } + + private ICodexInstance StartBinary(CodexStartupConfig config) + { + var portMap = new CodexPortMap(freePortFinder); + var factory = new CodexProcessRecipe(portMap, numberSource); + var recipe = factory.Initialize(config); + + var name = "codex_" + numberSource.GetNextNumber(); + var startInfo = new ProcessStartInfo( + fileName: recipe.Cmd, + arguments: recipe.Args + ); + //startInfo.UseShellExecute = true; + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + + var process = Process.Start(startInfo); + if (process == null || process.HasExited) + { + throw new Exception("Failed to start"); + } + + var local = "localhost"; + var instance = new CodexInstance( + name: name, + imageName: "binary", + startUtc: DateTime.UtcNow, + discoveryEndpoint: new Address("Disc", local, portMap.DiscPort), + apiEndpoint: new Address("Api", "http://" + local, portMap.ApiPort), + listenEndpoint: new Address("Listen", local, portMap.ListenPort), + ethAccount: null, + metricsEndpoint: null + ); + + var pc = new BinaryProcessControl(process, name); + processControlMap.Add(instance, pc); + + return instance; + } + + private void LogSeparator() + { + Log("----------------------------------------------------------------------------"); + } + + private void Log(string message) + { + pluginTools.GetLog().Log(message); } } } diff --git a/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs b/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs new file mode 100644 index 00000000..0e13fb35 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs @@ -0,0 +1,56 @@ +using System.Diagnostics; +using CodexClient; +using Logging; + +namespace CodexPlugin +{ + public class BinaryProcessControl : IProcessControl + { + private Process process; + private readonly string nodeName; + private bool running; + private readonly List logLines = new List(); + private readonly List streamTasks = new List(); + + public BinaryProcessControl(Process process, string nodeName) + { + running = true; + this.process = process; + this.nodeName = nodeName; + streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardOutput))); + streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardError))); + } + + private void ReadProcessStream(StreamReader reader) + { + while (running) + { + var line = reader.ReadLine(); + if (!string.IsNullOrEmpty(line)) logLines.Add(line); + } + } + + public void DeleteDataDirFolder() + { + throw new NotImplementedException(); + } + + public IDownloadedLog DownloadLog(LogFile file) + { + foreach (var line in logLines) file.WriteRaw(line); + return new DownloadedLog(file, nodeName); + } + + public bool HasCrashed() + { + return false; + } + + public void Stop(bool waitTillStopped) + { + running = false; + process.Kill(); + foreach (var t in streamTasks) t.Wait(); + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index e6c56f7c..1ebe6d77 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -15,7 +15,8 @@ namespace CodexPlugin public CodexPlugin(IPluginTools tools) { - codexStarter = new ContainerCodexStarter(tools, processControlMap); + //codexStarter = new ContainerCodexStarter(tools, processControlMap); + codexStarter = new BinaryCodexStarter(tools, processControlMap); codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory); this.tools = tools; } diff --git a/ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs b/ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs new file mode 100644 index 00000000..f439d640 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs @@ -0,0 +1,163 @@ +using System.Net.Sockets; +using System.Net; +using Utils; +using Nethereum.Util; + +namespace CodexPlugin +{ + public class ProcessRecipe + { + public ProcessRecipe(string cmd, string[] args) + { + Cmd = cmd; + Args = args; + } + + public string Cmd { get; } + public string[] Args { get; } + } + + public class CodexPortMap + { + public CodexPortMap(FreePortFinder freePortFinder) + { + ApiPort = freePortFinder.GetNextFreePort(); + DiscPort = freePortFinder.GetNextFreePort(); + ListenPort = freePortFinder.GetNextFreePort(); + } + + public int ApiPort { get; } + public int DiscPort { get; } + public int ListenPort { get; } + } + + public class CodexProcessRecipe + { + private readonly CodexPortMap portMap; + private readonly NumberSource numberSource; + + public CodexProcessRecipe(CodexPortMap portMap, NumberSource numberSource) + { + this.portMap = portMap; + this.numberSource = numberSource; + } + + public ProcessRecipe Initialize(CodexStartupConfig config) + { + args.Clear(); + + AddArg("--api-port", portMap.ApiPort); + AddArg("--api-bindaddr", "0.0.0.0"); + + var dataDir = $"datadir_{numberSource.GetNextNumber()}"; + AddArg("--data-dir", dataDir); + + AddArg("--disc-port", portMap.DiscPort); + AddArg("--log-level", config.LogLevelWithTopics()); + + // This makes the node announce itself to its local IP address. + var host = Dns.GetHostEntry(Dns.GetHostName()); + var addrs = host.AddressList.Where(a => a.AddressFamily == AddressFamily.InterNetwork).ToList(); + var ipaddrs = addrs.First(); + + AddArg("--nat", $"extip:{ipaddrs.ToStringInvariant()}"); + + AddArg("--listen-addrs", $"/ip4/0.0.0.0/tcp/{portMap.ListenPort}"); + + if (!string.IsNullOrEmpty(config.BootstrapSpr)) + { + AddArg("--bootstrap-node", config.BootstrapSpr); + } + if (config.StorageQuota != null) + { + AddArg("--storage-quota", config.StorageQuota.SizeInBytes.ToString()!); + } + if (config.BlockTTL != null) + { + AddArg("--block-ttl", config.BlockTTL.ToString()!); + } + if (config.BlockMaintenanceInterval != null) + { + AddArg("--block-mi", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString()); + } + if (config.BlockMaintenanceNumber != null) + { + AddArg("--block-mn", config.BlockMaintenanceNumber.ToString()!); + } + if (config.MetricsEnabled) + { + throw new Exception("Not supported"); + //var metricsPort = CreateApiPort(config, MetricsPortTag); + //AddEnvVar("CODEX_METRICS", "true"); + //AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); + //AddEnvVar("CODEX_METRICS_PORT", metricsPort); + //AddPodAnnotation("prometheus.io/scrape", "true"); + //AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString()); + } + + if (config.SimulateProofFailures != null) + { + throw new Exception("Not supported"); + //AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!); + } + + if (config.MarketplaceConfig != null) + { + throw new Exception("Not supported"); + //var mconfig = config.MarketplaceConfig; + //var gethStart = mconfig.GethNode.StartResult; + //var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag); + //var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress; + + //AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}"); + //AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress); + + //var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup; + + //// Custom scripting in the Codex test image will write this variable to a private-key file, + //// and pass the correct filename to Codex. + //var account = marketplaceSetup.EthAccountSetup.GetNew(); + //AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey); + //Additional(account); + + //SetCommandOverride(marketplaceSetup); + //if (marketplaceSetup.IsValidator) + //{ + // AddEnvVar("CODEX_VALIDATOR", "true"); + //} + } + + //if (!string.IsNullOrEmpty(config.NameOverride)) + //{ + // AddEnvVar("CODEX_NODENAME", config.NameOverride); + //} + + return Create(); + } + + private ProcessRecipe Create() + { + return new ProcessRecipe( + cmd: Path.Combine( + "d:", + "Dev", + "nim-codex", + "build", + "codex.exe" + ), + args: args.ToArray()); + } + + private readonly List args = new List(); + + private void AddArg(string arg, string val) + { + args.Add($"{arg}={val}"); + } + + private void AddArg(string arg, int val) + { + args.Add($"{arg}={val}"); + } + } +} diff --git a/ProjectPlugins/CodexPlugin/FreePortFinder.cs b/ProjectPlugins/CodexPlugin/FreePortFinder.cs new file mode 100644 index 00000000..9a67899d --- /dev/null +++ b/ProjectPlugins/CodexPlugin/FreePortFinder.cs @@ -0,0 +1,43 @@ +using System.Net.NetworkInformation; + +namespace CodexPlugin +{ + public class FreePortFinder + { + private readonly object _lock = new object(); + private int nextPort = 8080; + + public int GetNextFreePort() + { + lock (_lock) + { + return Next(); + } + } + + private int Next() + { + while (true) + { + var p = nextPort; + nextPort++; + + if (!IsInUse(p)) + { + return p; + } + + if (nextPort > 30000) throw new Exception("Running out of ports."); + } + } + + private bool IsInUse(int port) + { + var ipProps = IPGlobalProperties.GetIPGlobalProperties(); + if (ipProps.GetActiveTcpConnections().Any(t => t.LocalEndPoint.Port == port)) return true; + if (ipProps.GetActiveTcpListeners().Any(t => t.Port == port)) return true; + if (ipProps.GetActiveUdpListeners().Any(u => u.Port == port)) return true; + return false; + } + } +}