cs-codex-dist-tests/Tools/CodexNetDeployer/Deployer.cs

278 lines
10 KiB
C#
Raw Normal View History

2023-09-20 10:02:32 +00:00
using CodexContractsPlugin;
2023-10-24 08:29:07 +00:00
using CodexDiscordBotPlugin;
2023-09-20 10:02:32 +00:00
using CodexPlugin;
using Core;
using GethPlugin;
using KubernetesWorkflow.Types;
2023-06-26 13:37:16 +00:00
using Logging;
2023-09-20 10:02:32 +00:00
using MetricsPlugin;
2023-06-22 08:17:12 +00:00
namespace CodexNetDeployer
{
public class Deployer
{
private readonly Configuration config;
private readonly PeerConnectivityChecker peerConnectivityChecker;
2023-09-20 10:02:32 +00:00
private readonly EntryPoint entryPoint;
2023-09-28 09:31:09 +00:00
private readonly LocalCodexBuilder localCodexBuilder;
2023-06-22 08:17:12 +00:00
public Deployer(Configuration config)
{
this.config = config;
peerConnectivityChecker = new PeerConnectivityChecker();
2023-09-28 09:31:09 +00:00
localCodexBuilder = new LocalCodexBuilder(new ConsoleLog(), config.CodexLocalRepoPath);
2023-09-20 11:56:01 +00:00
ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
ProjectPlugin.Load<CodexContractsPlugin.CodexContractsPlugin>();
ProjectPlugin.Load<GethPlugin.GethPlugin>();
ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
2023-10-24 08:29:07 +00:00
ProjectPlugin.Load<CodexDiscordBotPlugin.CodexDiscordBotPlugin>();
2023-09-20 10:02:32 +00:00
entryPoint = CreateEntryPoint(new NullLog());
2023-06-22 08:17:12 +00:00
}
2023-09-20 10:02:32 +00:00
public void AnnouncePlugins()
2023-06-22 08:33:21 +00:00
{
2023-09-20 10:02:32 +00:00
var ep = CreateEntryPoint(new ConsoleLog());
2023-06-22 08:33:21 +00:00
2023-09-28 09:31:09 +00:00
localCodexBuilder.Intialize();
2023-09-20 10:02:32 +00:00
Log("Using plugins:" + Environment.NewLine);
var metadata = ep.GetPluginMetadata();
2023-09-21 06:49:09 +00:00
var longestKey = metadata.Keys.Max(k => k.Length);
2023-09-20 10:02:32 +00:00
foreach (var entry in metadata)
{
2023-09-21 06:49:09 +00:00
Console.Write(entry.Key);
Console.CursorLeft = longestKey + 5;
Console.WriteLine($"= {entry.Value}");
2023-09-20 10:02:32 +00:00
}
2023-09-20 10:02:32 +00:00
Log("");
}
2023-06-22 08:33:21 +00:00
2023-09-20 10:02:32 +00:00
public CodexDeployment Deploy()
{
2023-09-28 09:31:09 +00:00
localCodexBuilder.Build();
2023-09-20 10:02:32 +00:00
Log("Initializing...");
var startUtc = DateTime.UtcNow;
2023-09-20 10:02:32 +00:00
var ci = entryPoint.CreateInterface();
2023-06-22 08:33:21 +00:00
2023-09-20 10:02:32 +00:00
Log("Deploying Geth instance...");
2023-10-23 11:08:49 +00:00
var gethDeployment = DeployGeth(ci);
2023-09-20 10:02:32 +00:00
var gethNode = ci.WrapGethDeployment(gethDeployment);
2023-06-22 12:37:37 +00:00
2023-09-20 10:02:32 +00:00
Log("Geth started. Deploying Codex contracts...");
var contractsDeployment = ci.DeployCodexContracts(gethNode);
2023-10-30 12:30:14 +00:00
var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
2023-09-20 10:02:32 +00:00
Log("Codex contracts deployed.");
2023-06-22 12:37:37 +00:00
Log("Starting Codex nodes...");
2023-09-20 10:02:32 +00:00
var codexStarter = new CodexNodeStarter(config, ci, gethNode, contracts, config.NumberOfValidators!.Value);
var startResults = new List<CodexNodeStartResult>();
2023-06-22 12:37:37 +00:00
for (var i = 0; i < config.NumberOfCodexNodes; i++)
2023-06-22 08:33:21 +00:00
{
var result = codexStarter.Start(i);
if (result != null) startResults.Add(result);
2023-06-22 08:33:21 +00:00
}
2023-09-20 10:02:32 +00:00
Log("Codex nodes started.");
var metricsService = StartMetricsService(ci, startResults);
2023-07-11 08:59:41 +00:00
CheckPeerConnectivity(startResults);
CheckContainerRestarts(startResults);
var codexInstances = CreateCodexInstances(startResults);
2023-10-24 08:29:07 +00:00
var discordBotContainer = DeployDiscordBot(ci, gethDeployment, contractsDeployment);
2023-10-24 08:29:07 +00:00
return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService,
discordBotContainer, CreateMetadata(startUtc), config.DeployId);
2023-06-22 08:33:21 +00:00
}
2023-09-20 10:02:32 +00:00
private EntryPoint CreateEntryPoint(ILog log)
2023-06-22 08:17:12 +00:00
{
2023-06-27 06:29:39 +00:00
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
2023-09-20 10:02:32 +00:00
var configuration = new KubernetesWorkflow.Configuration(
kubeConfig,
operationTimeout: TimeSpan.FromMinutes(10),
2023-10-07 05:40:20 +00:00
retryDelay: TimeSpan.FromSeconds(10),
2023-09-20 10:02:32 +00:00
kubernetesNamespace: config.KubeNamespace);
var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet());
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata());
return result;
2023-06-22 08:33:21 +00:00
}
2023-10-23 11:08:49 +00:00
private GethDeployment DeployGeth(CoreInterface ci)
{
return ci.DeployGeth(s =>
{
s.IsMiner();
s.WithName("geth");
if (config.IsPublicTestNet)
{
s.AsPublicTestNet(new GethTestNetConfig(
2023-10-27 06:33:07 +00:00
discoveryPort: config.PublicGethDiscPort,
listenPort: config.PublicGethListenPort
));
}
2023-10-23 11:08:49 +00:00
});
}
2024-04-13 14:09:17 +00:00
private RunningPod? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
CodexContractsDeployment contractsDeployment)
{
if (!config.DeployDiscordBot) return null;
Log("Deploying Discord bot...");
var addr = gethDeployment.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag);
var info = new DiscordBotGethInfo(
host: addr.Host,
port: addr.Port,
privKey: gethDeployment.Account.PrivateKey,
marketplaceAddress: contractsDeployment.MarketplaceAddress,
tokenAddress: contractsDeployment.TokenAddress,
abi: contractsDeployment.Abi
);
var rc = ci.DeployCodexDiscordBot(new DiscordBotStartupConfig(
name: "discordbot-" + config.DeploymentName,
token: config.DiscordBotToken,
serverName: config.DiscordBotServerName,
adminRoleName: config.DiscordBotAdminRoleName,
adminChannelName: config.DiscordBotAdminChannelName,
kubeNamespace: config.KubeNamespace,
2024-02-19 08:11:36 +00:00
gethInfo: info,
rewardChannelName: config.DiscordBotRewardChannelName)
2023-10-25 09:53:33 +00:00
{
DataPath = config.DiscordBotDataPath
});
Log("Discord bot deployed.");
return rc;
}
2024-04-13 14:09:17 +00:00
private RunningPod? StartMetricsService(CoreInterface ci, List<CodexNodeStartResult> startResults)
2023-07-11 08:59:41 +00:00
{
if (!config.MetricsScraper || !startResults.Any()) return null;
2023-07-11 08:59:41 +00:00
Log("Starting metrics service...");
2023-08-13 07:07:23 +00:00
2023-09-20 10:02:32 +00:00
var runningContainer = ci.DeployMetricsCollector(startResults.Select(r => r.CodexNode).ToArray());
Log("Metrics service started.");
2023-08-13 07:07:23 +00:00
2023-09-20 10:02:32 +00:00
return runningContainer;
2023-07-11 08:59:41 +00:00
}
private CodexInstance[] CreateCodexInstances(List<CodexNodeStartResult> startResults)
{
2023-10-27 08:56:30 +00:00
// When freshly started, the Codex nodes are announcing themselves by an incorrect IP address.
// Only after fully initialized do they update to the provided NAT address.
// Therefore, we wait:
Thread.Sleep(TimeSpan.FromSeconds(5));
return startResults.Select(r => CreateCodexInstance(r.CodexNode)).ToArray();
}
private CodexInstance CreateCodexInstance(ICodexNode node)
{
2024-04-13 14:09:17 +00:00
return new CodexInstance(node.Container.RunningPod, node.GetDebugInfo());
}
2023-06-27 06:29:39 +00:00
private string? GetKubeConfig(string kubeConfigFile)
{
if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
return kubeConfigFile;
}
private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
{
if (!config.CheckPeerConnection || !codexContainers.Any()) return;
2023-09-20 10:02:32 +00:00
Log("Starting peer connectivity check for deployed nodes...");
peerConnectivityChecker.CheckConnectivity(codexContainers);
Log("Check passed.");
}
private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
{
var crashes = new List<RunningContainer>();
2023-09-20 10:02:32 +00:00
Log("Starting container crash check...");
foreach (var startResult in startResults)
{
2023-09-20 10:55:09 +00:00
var watcher = startResult.CodexNode.CrashWatcher;
if (watcher == null)
throw new Exception("Expected each CodexNode container to be created with a crash-watcher.");
2023-09-20 10:02:32 +00:00
if (watcher.HasContainerCrashed()) crashes.Add(startResult.CodexNode.Container);
}
if (!crashes.Any())
{
2023-09-20 10:02:32 +00:00
Log("Check passed.");
}
else
{
Log(
$"Check failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
throw new Exception("Deployment failed: One or more containers crashed.");
}
}
private DeploymentMetadata CreateMetadata(DateTime startUtc)
2023-06-23 07:08:18 +00:00
{
return new DeploymentMetadata(
2023-10-23 08:19:52 +00:00
name: config.DeploymentName,
startUtc: startUtc,
finishedUtc: DateTime.UtcNow,
2023-06-23 07:08:18 +00:00
kubeNamespace: config.KubeNamespace,
numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
numberOfValidators: config.NumberOfValidators!.Value,
storageQuotaMB: config.StorageQuota!.Value,
codexLogLevel: config.CodexLogLevel,
initialTestTokens: config.InitialTestTokens,
minPrice: config.MinPrice,
maxCollateral: config.MaxCollateral,
maxDuration: config.MaxDuration,
blockTTL: config.BlockTTL,
blockMI: config.BlockMI,
blockMN: config.BlockMN);
2023-06-23 07:08:18 +00:00
}
2023-06-22 08:33:21 +00:00
private void Log(string msg)
{
Console.WriteLine(msg);
2023-06-22 08:17:12 +00:00
}
}
public class FastHttpTimeSet : ITimeSet
{
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(2);
}
public int HttpMaxNumberOfRetries()
{
return 2;
}
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromSeconds(10);
}
public TimeSpan K8sOperationTimeout()
{
return TimeSpan.FromMinutes(10);
}
public TimeSpan WaitForK8sServiceDelay()
{
return TimeSpan.FromSeconds(30);
}
}
}