using CodexContractsPlugin; using CodexDiscordBotPlugin; using CodexPlugin; using Core; using GethPlugin; using KubernetesWorkflow.Types; using Logging; using MetricsPlugin; namespace CodexNetDeployer { public class Deployer { private readonly Configuration config; private readonly PeerConnectivityChecker peerConnectivityChecker; private readonly EntryPoint entryPoint; private readonly LocalCodexBuilder localCodexBuilder; public Deployer(Configuration config) { this.config = config; peerConnectivityChecker = new PeerConnectivityChecker(); localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath); ProjectPlugin.Load(); ProjectPlugin.Load(); ProjectPlugin.Load(); ProjectPlugin.Load(); ProjectPlugin.Load(); entryPoint = CreateEntryPoint(new NullLog()); } public void AnnouncePlugins() { var ep = CreateEntryPoint(new ConsoleLog()); localCodexBuilder.Intialize(); Log("Using plugins:" + Environment.NewLine); var metadata = ep.GetPluginMetadata(); var longestKey = metadata.Keys.Max(k => k.Length); foreach (var entry in metadata) { Console.Write(entry.Key); Console.CursorLeft = longestKey + 5; Console.WriteLine($"= {entry.Value}"); } Log(""); } public CodexDeployment Deploy() { localCodexBuilder.Build(); Log("Initializing..."); var startUtc = DateTime.UtcNow; var ci = entryPoint.CreateInterface(); Log("Deploying Geth instance..."); var gethDeployment = DeployGeth(ci); var gethNode = ci.WrapGethDeployment(gethDeployment); Log("Geth started. Deploying Codex contracts..."); var contractsDeployment = ci.DeployCodexContracts(gethNode); var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment); Log("Codex contracts deployed."); Log("Starting Codex nodes..."); var codexStarter = new CodexNodeStarter(config, ci, gethNode, contracts, config.NumberOfValidators!.Value); var startResults = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { var result = codexStarter.Start(i); if (result != null) startResults.Add(result); } Log("Codex nodes started."); var metricsService = StartMetricsService(ci, startResults); CheckPeerConnectivity(startResults); CheckContainerRestarts(startResults); var codexInstances = CreateCodexInstances(startResults); var discordBotContainer = DeployDiscordBot(ci, gethDeployment, contractsDeployment); return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService, discordBotContainer, CreateMetadata(startUtc), config.DeployId); } private EntryPoint CreateEntryPoint(ILog log) { var kubeConfig = GetKubeConfig(config.KubeConfigFile); var configuration = new KubernetesWorkflow.Configuration( kubeConfig, operationTimeout: TimeSpan.FromMinutes(10), retryDelay: TimeSpan.FromSeconds(10), kubernetesNamespace: config.KubeNamespace); var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet()); configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata()); return result; } private GethDeployment DeployGeth(CoreInterface ci) { return ci.DeployGeth(s => { s.IsMiner(); s.WithName("geth"); if (config.IsPublicTestNet) { s.AsPublicTestNet(new GethTestNetConfig( discoveryPort: config.PublicGethDiscPort, listenPort: config.PublicGethListenPort )); } }); } private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, CodexContractsDeployment contractsDeployment) { if (!config.DeployDiscordBot) return null; Log("Deploying Discord bot..."); var addr = gethDeployment.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag); var info = new DiscordBotGethInfo( host: addr.Host, port: addr.Port, privKey: gethDeployment.Account.PrivateKey, marketplaceAddress: contractsDeployment.MarketplaceAddress, tokenAddress: contractsDeployment.TokenAddress, abi: contractsDeployment.Abi ); var rc = ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( name: "discordbot-" + config.DeploymentName, token: config.DiscordBotToken, serverName: config.DiscordBotServerName, adminRoleName: config.DiscordBotAdminRoleName, adminChannelName: config.DiscordBotAdminChannelName, kubeNamespace: config.KubeNamespace, gethInfo: info, rewardChannelName: config.DiscordBotRewardChannelName) { DataPath = config.DiscordBotDataPath }); Log("Discord bot deployed."); return rc; } private RunningPod? StartMetricsService(CoreInterface ci, List startResults) { if (!config.MetricsScraper || !startResults.Any()) return null; Log("Starting metrics service..."); var runningContainer = ci.DeployMetricsCollector(scrapeInterval: TimeSpan.FromSeconds(10.0), startResults.Select(r => r.CodexNode).ToArray()); Log("Metrics service started."); return runningContainer; } private CodexInstance[] CreateCodexInstances(List startResults) { // When freshly started, the Codex nodes are announcing themselves by an incorrect IP address. // Only after fully initialized do they update to the provided NAT address. // Therefore, we wait: Thread.Sleep(TimeSpan.FromSeconds(5)); return startResults.Select(r => CreateCodexInstance(r.CodexNode)).ToArray(); } private CodexInstance CreateCodexInstance(ICodexNode node) { return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo()); } private string? GetKubeConfig(string kubeConfigFile) { if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; return kubeConfigFile; } private void CheckPeerConnectivity(List codexContainers) { if (!config.CheckPeerConnection || !codexContainers.Any()) return; Log("Starting peer connectivity check for deployed nodes..."); peerConnectivityChecker.CheckConnectivity(codexContainers); Log("Check passed."); } private void CheckContainerRestarts(List startResults) { var crashes = new List(); Log("Starting container crash check..."); foreach (var startResult in startResults) { var watcher = startResult.CodexNode.CrashWatcher; if (watcher == null) throw new Exception("Expected each CodexNode container to be created with a crash-watcher."); if (watcher.HasContainerCrashed()) crashes.Add(startResult.CodexNode.Container); } if (!crashes.Any()) { Log("Check passed."); } else { Log( $"Check failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}"); throw new Exception("Deployment failed: One or more containers crashed."); } } private DeploymentMetadata CreateMetadata(DateTime startUtc) { return new DeploymentMetadata( name: config.DeploymentName, startUtc: startUtc, finishedUtc: DateTime.UtcNow, kubeNamespace: config.KubeNamespace, numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, storageQuotaMB: config.StorageQuota!.Value, codexLogLevel: config.CodexLogLevel, initialTestTokens: config.InitialTestTokens, minPrice: config.MinPrice, maxCollateral: config.MaxCollateral, maxDuration: config.MaxDuration, blockTTL: config.BlockTTL, blockMI: config.BlockMI, blockMN: config.BlockMN); } private void Log(string msg) { Console.WriteLine(msg); } } public class FastHttpTimeSet : ITimeSet { public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromSeconds(2); } public TimeSpan HttpRetryTimeout() { return TimeSpan.FromSeconds(30); } public TimeSpan HttpCallTimeout() { return TimeSpan.FromSeconds(10); } public TimeSpan K8sOperationTimeout() { return TimeSpan.FromMinutes(10); } public TimeSpan K8sOperationRetryDelay() { return TimeSpan.FromSeconds(30); } } }